//*******************************************************************
// STATIC MEMBERS
//*******************************************************************
#region
public static GoogleCloudPubSubSinkState Create(GoogleCloudPubSubSinkOptions options, RollingFileSink errorsRollingFileSink)
{
if (options == null)
throw new ArgumentNullException("options");
else
return new GoogleCloudPubSubSinkState(options, errorsRollingFileSink);
}
//*******************************************************************
// CONSTRUCTOR
//*******************************************************************
#region
/// <summary>
/// Construct a sink that saves logs to the specified Google PubSub account.
/// </summary>
/// <param name="options">Options configuring how the sink behaves, may NOT be null</param>
public PeriodicGoogleCloudPubSubSink(GoogleCloudPubSubSinkOptions options )
: base(options.BatchPostingLimit, options.Period)
{
this._state = GoogleCloudPubSubSinkState.Create(options, null);
}
//*******************************************************************
// CONSTRUCTOR
//*******************************************************************
#region
private GoogleCloudPubSubSinkState(GoogleCloudPubSubSinkOptions options, RollingFileSink errorsRollingFileSink)
{
//--- Mandatory options validations --------------------
if (options.BatchPostingLimit < 1 ) throw new ArgumentException("batchPostingLimit must be >= 1");
if (string.IsNullOrWhiteSpace(options.ProjectId)) throw new ArgumentException("options.ProjectId");
if (string.IsNullOrWhiteSpace(options.TopicId)) throw new ArgumentException("options.TopicId");
//---
// All is ok ...
this._options = options;
this._errorsRollingFileSink = errorsRollingFileSink;
this._periodicBatchingFormatter = options.CustomFormatter ?? new GoogleCloudPubSubRawFormatter();
this._durableFormatter = options.CustomFormatter ?? new GoogleCloudPubSubRawFormatter();
this._topic = PublisherClient.FormatTopicName(options.ProjectId, options.TopicId);
this._client = PublisherClient.Create();
//---
try
{
if (!string.IsNullOrWhiteSpace(this.Options.EventFieldSeparator) && !string.IsNullOrWhiteSpace(this.Options.MessageAttrMinValue))
{
string[] auxArray = this.Options.MessageAttrMinValue.Split(new char[] { '#' }, StringSplitOptions.RemoveEmptyEntries);
if (auxArray.Length == 2)
{
this._attrMinPosition = Int32.Parse(auxArray[0]);
this._attrMinName = auxArray[1];
this._attrMinCreate = true;
}
}
}
catch
{
this._attrMinCreate = false;
}
}
/// <summary>
/// Constructor specifying concret options values.
/// </summary>
/// <param name="projectId">Google Cloud PubSub Project ID</param>
/// <param name="topicId">Google Cloud PubSub Topic ID</param>
/// <param name="bufferBaseFilename">Path to directory and file name prefix that can be used as a log shipping buffer for increasing the reliability of the log forwarding.</param>
/// <param name="bufferFileSizeLimitBytes">The maximum size, in bytes, to which the buffer file for a specific date will be allowed to grow.
/// Once the limit is reached no more events will be stored. Pass null for default value.</param>
/// <param name="bufferLogShippingIntervalMilisec">The interval, in miliseconds, between checking the buffer files. Pass null for default value.</param>
/// <param name="bufferRetainedFileCountLimit">The maximum number of buffer files that will be retained, including the current buffer file. Pass null for default value (no limit). The minimum value is 2.</param>
/// <param name="bufferFileExtension">The file extension to use with buffer files. Pass null for default value.</param>
/// <param name="batchPostingLimit">The maximum number of events to post in a single batch. Pass null for default value.</param>
/// <param name="batchSizeLimitBytes">The maximum size, in bytes, of the batch to send to PubSub. By default no limit will be applied.</param>
/// <param name="minimumLogEventLevel">The minimum log event level required in order to write an event to the sink. Pass null for default value (minimum).</param>
/// <param name="errorBaseFilename">Path to directory that can be used as a log shipping for storing internal errors.
/// If set then it means we want to store errors. It can be used the same path as the buffer log (bufferBaseFilename) but the file name can't start with the same string.</param>
/// <param name="errorFileSizeLimitBytes">The maximum size, in bytes, to which the error file for a specific date will be allowed to grow. By default no limit will be applied.</param>
/// <param name="errorStoreEvents">If set to 'true' then events related to any error will be saved to the error file (after the error message). Pass null for default value (false).</param>
/// <param name="debugStoreBatchLimitsOverflows">If set to 'true' then overflows when creating batch posts will be stored (overflows for BatchPostingLimit and also for BatchSizeLimitBytes). Pass null for default value (false).</param>
/// <param name="debugStoreAll">If set to 'true' then debug data will be stored. Pass null for default value (false).</param>
/// <param name="messageDataToBase64">If set to 'true' then data on PubSub messages is converted to Base64. Pass null for default value (true).</param>
/// <param name="eventFieldSeparator">Fields separator in event data.</param>
/// <param name="messageAttrMinValue">If given indicates that the PubSub message has to contain an attribute that is obtained as the MIN value for a concret field in the event dada.</param>
/// <param name="messageAttrFixed">If given then in each message to PubSub will be added as many attributes as elements has de dictionary, where
/// the key corresponds to an attribute name and the value corresponds to its value to set.</param>
/// <param name="debugStoreEventSkip">If set to 'true' then skiped events (greater than the BatchSizeLimitBytes) will be stored.</param>
/// <param name="bufferRollingSpecifier">Rolling specifier: {Date}, {Hour} or {HalfHour}. The default one is {Hour}.</param>
/// <param name="errorRetainedFileCountLimit">The maximum number of error log files that will be retained, including the current error file. For unlimited retention, pass null. The default is 31.</param>
/// <param name="debugStoreFileAction">If set to 'true' then all file actions(move forward, delete, ...) will me stored.</param>
/// <param name="errorRollingSpecifier">Rolling specifier: {Date}, {Hour} or {HalfHour}. The default one is {Date}.</param>
/// <returns>LoggerConfiguration object</returns>
/// <exception cref="ArgumentNullException"><paramref name="projectId"/> is <see langword="null" />.</exception>
/// <exception cref="ArgumentNullException"><paramref name="topicId"/> is <see langword="null" />.</exception>
/// <exception cref="ArgumentNullException"><paramref name="bufferBaseFilename"/> is <see langword="null" />.</exception>
public DurableGoogleCloudPubSubSink(
string projectId,
string topicId,
string bufferBaseFilename,
long? bufferFileSizeLimitBytes = null,
int? bufferLogShippingIntervalMilisec = null,
int? bufferRetainedFileCountLimit = null,
string bufferFileExtension = null,
int? batchPostingLimit = null,
long? batchSizeLimitBytes = null,
LogEventLevel minimumLogEventLevel = LevelAlias.Minimum,
string errorBaseFilename = null,
long? errorFileSizeLimitBytes = null,
bool? errorStoreEvents = null,
bool? debugStoreBatchLimitsOverflows = null,
bool? debugStoreAll = null,
bool? messageDataToBase64 = null,
string eventFieldSeparator = null,
string messageAttrMinValue = null,
Dictionary<string, string> messageAttrFixed = null,
bool? debugStoreEventSkip = null,
string bufferRollingSpecifier = null,
int? errorRetainedFileCountLimit = null,
bool? debugStoreFileAction = null,
string errorRollingSpecifier = null)
{
//--- Creating an options object with the received parameters -------------
// If a parameter is null then the corresponding option will not be set and it will be used its default value.
GoogleCloudPubSubSinkOptions options = new GoogleCloudPubSubSinkOptions(projectId, topicId);
options.SetValues(
bufferBaseFilename,
bufferFileSizeLimitBytes,
bufferLogShippingIntervalMilisec,
bufferRetainedFileCountLimit,
bufferFileExtension,
batchPostingLimit,
batchSizeLimitBytes,
minimumLogEventLevel,
errorBaseFilename,
errorFileSizeLimitBytes,
errorStoreEvents,
debugStoreBatchLimitsOverflows,
debugStoreAll,
messageDataToBase64,
eventFieldSeparator,
messageAttrMinValue,
messageAttrFixed,
debugStoreEventSkip,
bufferRollingSpecifier,
errorRetainedFileCountLimit,
debugStoreFileAction,
errorRollingSpecifier);
//-----
this.Initialize(options);
}
private GoogleCloudPubSubSinkState _state; // -> Contains the options.
#endregion Fields
#region Constructors
/// <summary>
/// Default constructor with a given options object.
/// </summary>
/// <param name="options"></param>
public DurableGoogleCloudPubSubSink(GoogleCloudPubSubSinkOptions options)
{
this.Initialize(options);
}
//--------------
private void ValidateMandatoryOptions(GoogleCloudPubSubSinkOptions options)
{
if (string.IsNullOrEmpty(options.ProjectId))
{
throw new ArgumentNullException(nameof(options.ProjectId), "No project specified.");
}
if (string.IsNullOrEmpty(options.TopicId))
{
throw new ArgumentNullException(nameof(options.TopicId), "No topic specified.");
}
if (string.IsNullOrWhiteSpace(options.BufferBaseFilename))
{
throw new ArgumentException("Cannot create the durable GoogleCloudPubSub sink without BufferBaseFilename");
}
if (string.IsNullOrWhiteSpace(options.BufferFileExtension))
{
throw new ArgumentException("Cannot create the durable GoogleCloudPubSub sink without BufferFileExtension");
}
if (!options.BufferLogShippingInterval.HasValue)
{
throw new ArgumentException("Cannot create the durable GoogleCloudPubSub sink without BufferLogShippingInterval");
}
if (options.BufferRetainedFileCountLimit.HasValue && options.BufferRetainedFileCountLimit.Value < 2)
{
throw new ArgumentException("BufferRetainedFileCountLimit minimum value is 2");
}
}
//--------------
private void Initialize(GoogleCloudPubSubSinkOptions options)
{
//--- Mandatory options validations --------------------
this.ValidateMandatoryOptions(options);
//---
// All is ok ... instances are created using the defined options...
if (!options.BufferFileExtension.StartsWith("."))
options.BufferFileExtension = "." + options.BufferFileExtension;
//--- RollingFileSink to store internal errors ------------------
// It will be generated a file for each day.
string errorsFileExtension = (options.BufferFileExtension == ".log" ? ".errorlog" : ".log");
if (!string.IsNullOrWhiteSpace(options.ErrorBaseFilename))
{
this._errorsRollingFileSink = new RollingFileSink(
options.ErrorBaseFilename + CNST_Specifier_Separator + options.ErrorRollingSpecifier + errorsFileExtension,
new GoogleCloudPubSubRawFormatter(), // Formatter for error info (raw).
options.ErrorFileSizeLimitBytes,
options.ErrorRetainedFileCountLimit
);
}
//---
this._state = GoogleCloudPubSubSinkState.Create(options, this._errorsRollingFileSink);
this._shipper = new GoogleCloudPubSubLogShipper(this._state);
//---
//--- RollingFileSink to store data to be sent to PubSub ------------------
// It will be generated a file for each day.
this._dataRollingFileSink = new RollingFileSink(
options.BufferBaseFilename + CNST_Specifier_Separator + options.BufferRollingSpecifier + options.BufferFileExtension,
this._state.DurableFormatter, // Formatter for data to insert into the buffer file.
options.BufferFileSizeLimitBytes,
null
);
// NOTE: if the encoding is set to UTF8 then the BOM is inserted into the file and has to be
// taken in mind when reading from the file.
}
private static void ValidateMandatoryOptions(GoogleCloudPubSubSinkOptions options)
{
//--- Mandatory data for any type of sink used (durable or not). -------------------
if (string.IsNullOrEmpty(options.ProjectId))
throw new ArgumentNullException(nameof(options.ProjectId), "No project specified.");
if (string.IsNullOrEmpty(options.TopicId))
throw new ArgumentNullException(nameof(options.TopicId), "No topic specified.");
//--------------------------------------------------------------------
// At this version we just support a durable sink !!!!
if (string.IsNullOrEmpty(options.BufferBaseFilename))
throw new ArgumentNullException(nameof(options.BufferBaseFilename), "This version just supports a durable sink. No buffer specified.");
//--------------------------------------------------------------------
}
//--------------------------------------
/// <summary>
/// Overload to use an specific configuration.
///
/// By passing in the options object the BufferBaseFilename, you make this into a durable sink.
/// Meaning it will log to disk first and tries to deliver to the PubSub server in the background.
/// </summary>
/// <remarks>
/// </remarks>
/// <param name="loggerSinkConfiguration">Options for the sink.</param>
/// <param name="options">Provides options specific to the GoogleCloudPubSub sink</param>
/// <returns>LoggerConfiguration object</returns>
/// <exception cref="ArgumentNullException"><paramref name="options"/> is <see langword="null" />.</exception>
public static LoggerConfiguration GoogleCloudPubSub(
this LoggerSinkConfiguration loggerSinkConfiguration,
GoogleCloudPubSubSinkOptions options)
{
//--- Mandatory parameters ------------
ValidateMandatoryOptions(options);
//---
// All is ok ...
// -If we are given a BufferBaseFilename that means that log events have to be stored
// on a buffer file on disk and then sent to PubSub in groups deppending on a timer (in the background).
// In this case the used sink is : DurableGoogleCloudPubSubSink
//
// -If we are not given a BufferBaseFilename that means that log events have to be stored
// in memory.
// In this case the used sink is : GoogleCloudPubSubSink
//
// The choosen sink will contain the execution options.
//
var sink = string.IsNullOrWhiteSpace(options.BufferBaseFilename)
? (ILogEventSink)new PeriodicGoogleCloudPubSubSink(options)
: new DurableGoogleCloudPubSubSink(options);
return loggerSinkConfiguration.Sink(sink, options.MinimumLogEventLevel ?? LevelAlias.Minimum);
}
