/// <summary>
/// Initializes a new instance of the <see cref="WindowsAzureTableSink"/> class with the specified connection string and table address.
/// </summary>
/// <param name="instanceName">The name of the instance originating the entries.</param>
/// <param name="connectionString">The connection string for the storage account.</param>
/// <param name="tableAddress">Either the name of the table, or the absolute URI to the table.</param>
/// <param name="bufferInterval">The buffering interval to wait for events to accumulate before sending them to Azure Storage.</param>
/// <param name="maxBufferSize">The maximum number of entries that can be buffered while it's sending to Azure Storage before the sink starts dropping entries.</param>
/// <param name="onCompletedTimeout">Defines a timeout interval for when flushing the entries after an <see cref="OnCompleted"/> call is received and before disposing the sink.
/// This means that if the timeout period elapses, some event entries will be dropped and not sent to the store. Normally, calling <see cref="IDisposable.Dispose"/> on
/// the <see cref="System.Diagnostics.Tracing.EventListener"/> will block until all the entries are flushed or the interval elapses.
/// If <see langword="null"/> is specified, then the call will block indefinitely until the flush operation finishes.</param>
public WindowsAzureTableSink(string instanceName, string connectionString, string tableAddress, TimeSpan bufferInterval, int maxBufferSize, TimeSpan onCompletedTimeout)
{
Guard.ArgumentNotNullOrEmpty(instanceName, "instanceName");
Guard.ArgumentNotNullOrEmpty(connectionString, "connectionString");
Guard.ArgumentNotNullOrEmpty(tableAddress, "tableAddress");
Guard.ArgumentIsValidTimeout(onCompletedTimeout, "onCompletedTimeout");
this.onCompletedTimeout = onCompletedTimeout;
CloudStorageAccount account = GetStorageAccount(connectionString);
if (!IsValidTableName(tableAddress))
{
throw new ArgumentException(WindowsAzureResources.InvalidTableName, "tableAddress");
}
this.instanceName = NormalizeInstanceName(instanceName);
this.client = account.CreateCloudTableClient();
this.client.DefaultRequestOptions = new TableRequestOptions
{
RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(5), 7)
};
this.table = this.client.GetTableReference(tableAddress);
string sinkId = string.Format(CultureInfo.InvariantCulture, "WindowsAzureTableSink ({0})", instanceName);
this.bufferedPublisher = BufferedEventPublisher <CloudEventEntry> .CreateAndStart(sinkId, this.PublishEventsAsync, bufferInterval, BufferCountTrigger, maxBufferSize, this.cancellationTokenSource.Token);
}
/// <summary>
/// Initializes a new instance of the Seq sink.
/// </summary>
/// <param name="serverUrl">The base URL of the Seq server that log events will be written to.</param>
/// <param name="apiKey">A Seq <i>API key</i> that authenticates the client to the Seq server.</param>
/// <param name="bufferingInterval">The buffering interval between each batch publishing. Default value is <see cref="Buffering.DefaultBufferingInterval" />.</param>
/// <param name="onCompletedTimeout">Time limit for flushing the entries after an <see cref="SeqSink.OnCompleted" /> call is received.</param>
/// <param name="bufferingCount">Number of entries that will trigger batch publishing. Default is <see cref="Buffering.DefaultBufferingCount" /></param>
/// <param name="maxBufferSize">The maximum number of entries that can be buffered before the sink starts dropping entries.
/// This means that if the timeout period elapses, some event entries will be dropped and not sent to the store. Normally, calling <see cref="IDisposable.Dispose" /> on
/// the <see cref="System.Diagnostics.Tracing.EventListener" /> will block until all the entries are flushed or the interval elapses.
/// If <see langword="null" /> is specified, then the call will block indefinitely until the flush operation finishes.</param>
public SeqSink(
string serverUrl,
string apiKey,
TimeSpan bufferingInterval,
int bufferingCount,
int maxBufferSize,
TimeSpan onCompletedTimeout)
{
if (serverUrl == null)
{
throw new ArgumentNullException("serverUrl");
}
var baseUri = serverUrl;
if (!baseUri.EndsWith("/"))
{
baseUri += "/";
}
_serverUrl = baseUri;
_apiKey = apiKey;
_onCompletedTimeout = onCompletedTimeout;
_bufferedPublisher = BufferedEventPublisher <EventEntry> .CreateAndStart("Seq", PublishEventsAsync, bufferingInterval,
bufferingCount, maxBufferSize, _cancellationTokenSource.Token);
}
/// <summary>
/// Initializes a new instance of the <see cref="ElasticsearchSink"/> class with the specified connection string and table address.
/// </summary>
/// <param name="instanceName">The name of the instance originating the entries.</param>
/// <param name="connectionString">The connection string for the storage account.</param>
/// <param name="index">Index name prefix formatted as index-{0:yyyy.MM.DD}.</param>
/// <param name="type">Elasticsearch entry type.</param>
/// <param name="flattenPayload">Flatten the payload collection when serializing event entries</param>
/// <param name="bufferInterval">The buffering interval to wait for events to accumulate before sending them to Elasticsearch.</param>
/// <param name="bufferingCount">The buffering event entry count to wait before sending events to Elasticsearch </param>
/// <param name="maxBufferSize">The maximum number of entries that can be buffered while it's sending to Windows Azure Storage before the sink starts dropping entries.</param>
/// <param name="onCompletedTimeout">Defines a timeout interval for when flushing the entries after an <see cref="OnCompleted"/> call is received and before disposing the sink.
/// This means that if the timeout period elapses, some event entries will be dropped and not sent to the store. Normally, calling <see cref="IDisposable.Dispose"/> on
/// the <see cref="System.Diagnostics.Tracing.EventListener"/> will block until all the entries are flushed or the interval elapses.
/// If <see langword="null"/> is specified, then the call will block indefinitely until the flush operation finishes.</param>
/// <param name="jsonGlobalContextExtension">A json encoded key/value set of global environment parameters to be included in each log entry</param>
public ElasticsearchSink(string instanceName, string connectionString, string index, string type, bool?flattenPayload, TimeSpan bufferInterval,
int bufferingCount, int maxBufferSize, TimeSpan onCompletedTimeout, string jsonGlobalContextExtension = null)
{
Guard.ArgumentNotNullOrEmpty(instanceName, "instanceName");
Guard.ArgumentNotNullOrEmpty(connectionString, "connectionString");
Guard.ArgumentNotNullOrEmpty(index, "index");
Guard.ArgumentNotNullOrEmpty(type, "type");
Guard.ArgumentIsValidTimeout(onCompletedTimeout, "onCompletedTimeout");
Guard.ArgumentGreaterOrEqualThan(0, bufferingCount, "bufferingCount");
if (Regex.IsMatch(index, "[\\\\/*?\",<>|\\sA-Z]"))
{
throw new ArgumentException(Resource.InvalidElasticsearchIndexNameError, "index");
}
this.onCompletedTimeout = onCompletedTimeout;
this.instanceName = instanceName;
this.flattenPayload = flattenPayload ?? true;
this.elasticsearchUrl = new Uri(new Uri(connectionString), BulkServiceOperationPath);
this.index = index;
this.type = type;
var sinkId = string.Format(CultureInfo.InvariantCulture, "ElasticsearchSink ({0})", instanceName);
bufferedPublisher = BufferedEventPublisher <EventEntry> .CreateAndStart(sinkId, PublishEventsAsync, bufferInterval,
bufferingCount, maxBufferSize, cancellationTokenSource.Token);
this._jsonGlobalContextExtension = !string.IsNullOrEmpty(jsonGlobalContextExtension)? JsonConvert.DeserializeObject <Dictionary <string, string> >(jsonGlobalContextExtension): null;
}
protected override void Given()
{
this.ShouldThrowOnPublish = true;
this.bufferingCount = 10;
this.totalEventsToPublish = 0;
var automaticFlushingInterval = TimeSpan.FromMilliseconds(1);
this.sut = BufferedEventPublisher <int> .CreateAndStart("sink", PublishEventsAsync, automaticFlushingInterval, bufferingCount, 1000, new CancellationToken());
}
protected override void Given()
{
this.bufferingCount = 10;
this.totalEventsToPublish = 0;
var automaticFlushingInterval = TimeSpan.FromMinutes(30);
this.sut = BufferedEventPublisher <int> .CreateAndStart("sink", PublishEventsAsync, automaticFlushingInterval, bufferingCount, 1000, new CancellationToken());
Thread.Sleep(200);
}
private void SetupSink(TimeSpan bufferingInterval, int bufferingCount, int maxBufferSize, TimeSpan onCompletedTimeout, string partitionKey)
{
useAutomaticSizedBuffer = bufferingCount == 0;
this.partitionKey = partitionKey;
this.onCompletedTimeout = onCompletedTimeout;
sinkId = string.Format(CultureInfo.InvariantCulture, "EventHubAmqpSink ({0})", Guid.NewGuid());
bufferedPublisher = BufferedEventPublisher <EventEntry> .CreateAndStart(sinkId, PublishEventsAsync, bufferingInterval,
bufferingCount, maxBufferSize, cancellationTokenSource.Token);
}
/// <summary>
/// Initializes a new instance of the <see cref="SqlDatabaseSink" /> class with the specified instance name, connection string and table name.
/// </summary>
/// <param name="instanceName">The name of the instance originating the entries.</param>
/// <param name="connectionString">The connection string.</param>
/// <param name="tableName">The name of the table.</param>
/// <param name="bufferingInterval">The buffering interval between each batch publishing.</param>
/// <param name="bufferingCount">The number of entries that will trigger a batch publishing.</param>
public CustomSqlSink(string instanceName, string connectionString, string tableName = "Traces", TimeSpan?bufferingInterval = null, int bufferingCount = Buffering.DefaultBufferingCount)
{
Guard.ArgumentNotNullOrEmpty(instanceName, "instanceName");
ValidateSqlConnectionString(connectionString, "connectionString");
string sinkId = string.Format(CultureInfo.InvariantCulture, "DatabaseSink ({0})", instanceName);
this.dbProviderFactory = SqlClientFactory.Instance;
this.bufferedPublisher = new BufferedEventPublisher <EventRecord>(sinkId, this.PublishEventsAsync, bufferingInterval ?? Buffering.DefaultBufferingInterval, bufferingCount, 30000, this.cancellationTokenSource.Token);
this.instanceName = instanceName;
this.connectionString = connectionString;
this.tableName = tableName ?? "Traces";
}
/// <summary>
/// Initializes a new instance of the <see cref="SqlDatabaseSink" /> class with the specified instance name, connection string and table name.
/// </summary>
/// <param name="instanceName">The name of the instance originating the entries.</param>
/// <param name="connectionString">The connection string.</param>
/// <param name="tableName">The name of the table.</param>
/// <param name="bufferingInterval">The buffering interval between each batch publishing.</param>
/// <param name="bufferingCount">The number of entries that will trigger a batch publishing.</param>
/// <param name="maxBufferSize">The maximum number of entries that can be buffered while it's sending to the store before the sink starts dropping entries.</param>
/// <param name="onCompletedTimeout">Defines a timeout interval for when flushing the entries after an <see cref="OnCompleted"/> call is received and before disposing the sink.
/// This means that if the timeout period elapses, some event entries will be dropped and not sent to the store. Normally, calling <see cref="IDisposable.Dispose"/> on
/// the <see cref="Microsoft.Diagnostics.Tracing.EventListener"/> will block until all the entries are flushed or the interval elapses.
/// If <see langword="null"/> is specified, then the call will block indefinitely until the flush operation finishes.</param>
public SqlDatabaseSink(string instanceName, string connectionString, string tableName, TimeSpan bufferingInterval, int bufferingCount, int maxBufferSize, TimeSpan onCompletedTimeout)
{
Guard.ArgumentNotNullOrEmpty(instanceName, "instanceName");
Guard.ArgumentNotNullOrEmpty(connectionString, "connectionString");
Guard.ArgumentNotNullOrEmpty(tableName, "tableName");
Guard.ArgumentIsValidTimeout(onCompletedTimeout, "onCompletedTimeout");
ValidateSqlConnectionString(connectionString, "connectionString");
this.instanceName = instanceName;
this.connectionString = connectionString;
this.tableName = tableName;
this.onCompletedTimeout = onCompletedTimeout;
this.retryPolicy.Retrying += Retrying;
string sinkId = string.Format(CultureInfo.InvariantCulture, "SqlDatabaseSink ({0})", instanceName);
this.bufferedPublisher = BufferedEventPublisher <EventEntry> .CreateAndStart(sinkId, this.PublishEventsAsync, bufferingInterval, bufferingCount, maxBufferSize, this.cancellationTokenSource.Token);
}
public void Dispose()
{
if (_cancellationTokenSource != null)
{
_cancellationTokenSource.Cancel();
_cancellationTokenSource.Dispose();
_cancellationTokenSource = null;
}
if (_bufferedPublisher != null)
{
_bufferedPublisher.Dispose();
_bufferedPublisher = null;
}
if (_BQSvc != null)
{
_BQSvc.Dispose();
_BQSvc = null;
}
}
internal EventHubHttpSink(IHttpClient httpClient, string eventHubNamespace, string eventHubName, string publisherId, string sasToken, TimeSpan bufferingInterval, int bufferingCount, int maxBufferSize, TimeSpan onCompletedTimeout)
{
Guard.ArgumentNotNullOrEmpty(eventHubNamespace, "eventHubConnectionString");
Guard.ArgumentNotNullOrEmpty(eventHubName, "eventHubName");
Guard.ArgumentNotNullOrEmpty(publisherId, "partitionKey");
Guard.ArgumentNotNullOrEmpty(sasToken, "sasToken");
useAutomaticSizedBuffer = bufferingCount == 0;
this.eventHubNamespace = eventHubNamespace;
this.eventHubName = eventHubName;
this.publisherId = publisherId;
this.onCompletedTimeout = onCompletedTimeout;
sinkId = string.Format(CultureInfo.InvariantCulture, "EventHubHttpSink ({0})", Guid.NewGuid());
bufferedPublisher = BufferedEventPublisher <EventEntry> .CreateAndStart(sinkId, PublishEventsAsync, bufferingInterval, bufferingCount, maxBufferSize, cancellationTokenSource.Token);
this.httpClient = httpClient;
httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Authorization", sasToken);
}
/// <summary>
/// Initializes a new instance of the <see cref="MongoSink" /> class.
/// </summary>
/// <param name="client">
/// The client.
/// </param>
/// <param name="instanceName">
/// The instance name.
/// </param>
/// <param name="collectionName">
/// The collection name.
/// </param>
/// <param name="bufferingInterval">
/// The buffering interval.
/// </param>
/// <param name="bufferingCount">
/// The buffering count.
/// </param>
/// <param name="maxBufferSize">
/// The max buffer size.
/// </param>
/// <param name="onCompletedTimeout">
/// The on completed timeout.
/// </param>
public MongoSink(
IMongoClient client,
string instanceName,
string collectionName,
TimeSpan bufferingInterval,
int bufferingCount,
int maxBufferSize,
TimeSpan onCompletedTimeout)
{
_onCompletedTimeout = onCompletedTimeout;
_bufferedPublisher = BufferedEventPublisher <EventEntry> .CreateAndStart(
"mongodb",
PublishEventsAsync,
bufferingInterval,
bufferingCount,
maxBufferSize,
_cancellationTokenSource.Token);
_collection = client.GetDatabase(instanceName).GetCollection <BsonDocument>(collectionName);
}
/// <summary>
/// Initializes a new instance of the <see cref="LogglySink"/> class with the specified connection string and table address.
/// </summary>
/// <param name="instanceName">The name of the instance originating the entries.</param>
/// <param name="connectionString">The connection string for the storage account.</param>
/// <param name="customerToken">The loggly customerToken that must be part of the Url.</param>
/// <param name="tag">The tag used in loggly updates. Default is to use the instanceName.</param>
/// <param name="flattenPayload">Flatten the payload if you want the parameters serialized.</param>
/// <param name="bufferInterval">The buffering interval to wait for events to accumulate before sending them to Loggly.</param>
/// <param name="bufferingCount">The buffering event entry count to wait before sending events to Loggly </param>
/// <param name="maxBufferSize">The maximum number of entries that can be buffered while it's sending to Windows Azure Storage before the sink starts dropping entries.</param>
/// <param name="onCompletedTimeout">Defines a timeout interval for when flushing the entries after an <see cref="OnCompleted"/> call is received and before disposing the sink.
/// This means that if the timeout period elapses, some event entries will be dropped and not sent to the store. Normally, calling <see cref="IDisposable.Dispose"/> on
/// the <see cref="System.Diagnostics.Tracing.EventListener"/> will block until all the entries are flushed or the interval elapses.
/// If <see langword="null"/> is specified, then the call will block indefinitely until the flush operation finishes.</param>
public LogglySink(string instanceName, string connectionString, string customerToken, string tag, bool flattenPayload, TimeSpan bufferInterval,
int bufferingCount, int maxBufferSize, TimeSpan onCompletedTimeout)
{
Guard.ArgumentNotNullOrEmpty(instanceName, "instanceName");
Guard.ArgumentNotNullOrEmpty(connectionString, "connectionString");
Guard.ArgumentNotNullOrEmpty(customerToken, "index");
Guard.ArgumentIsValidTimeout(onCompletedTimeout, "onCompletedTimeout");
Guard.ArgumentGreaterOrEqualThan(0, bufferingCount, "bufferingCount");
this.onCompletedTimeout = onCompletedTimeout;
this.instanceName = instanceName;
this.logglyUrl = new Uri(new Uri(connectionString), string.Format(BulkServiceOperationPath, customerToken, tag));
this.customerToken = customerToken;
this.tag = string.IsNullOrWhiteSpace(tag) ? instanceName : tag;
this.flattenPayload = flattenPayload;
var sinkId = string.Format(CultureInfo.InvariantCulture, "LogglySink ({0})", instanceName);
bufferedPublisher = BufferedEventPublisher <EventEntry> .CreateAndStart(sinkId, PublishEventsAsync, bufferInterval,
bufferingCount, maxBufferSize, cancellationTokenSource.Token);
}
/// <summary>
/// Initializes a new instance of the <see cref="BlueprintSqlDatabaseSink" /> class with the specified instance name, connection string, table name and stored procedure name.
/// </summary>
/// <param name="instanceName">The name of the instance originating the entries.</param>
/// <param name="connectionString">The connection string.</param>
/// <param name="tableName">The name of the table.</param>
/// <param name="storedProcedureName">The name of the stored procedure that writes to the table.</param>
/// <param name="bufferingInterval">The buffering interval between each batch publishing.</param>
/// <param name="bufferingCount">The number of entries that will trigger a batch publishing.</param>
/// <param name="maxBufferSize">The maximum number of entries that can be buffered while it's sending to the store before the sink starts dropping entries.</param>
/// <param name="onCompletedTimeout">Defines a timeout interval for when flushing the entries after an <see cref="OnCompleted"/> call is received and before disposing the sink.
/// This means that if the timeout period elapses, some event entries will be dropped and not sent to the store. Normally, calling <see cref="IDisposable.Dispose"/> on
/// the <see cref="System.Diagnostics.Tracing.EventListener"/> will block until all the entries are flushed or the interval elapses.
/// If <see langword="null"/> is specified, then the call will block indefinitely until the flush operation finishes.</param>
public BlueprintSqlDatabaseSink(string instanceName, string connectionString, string tableName, string storedProcedureName, TimeSpan bufferingInterval, int bufferingCount, int maxBufferSize, TimeSpan onCompletedTimeout)
{
Guard.ArgumentNotNullOrEmpty(instanceName, "instanceName");
Guard.ArgumentNotNullOrEmpty(connectionString, "connectionString");
Guard.ArgumentNotNullOrEmpty(tableName, "tableName");
Guard.ArgumentNotNullOrEmpty(storedProcedureName, "storedProcedureName");
Guard.ArgumentIsValidTimeout(onCompletedTimeout, "onCompletedTimeout");
Guard.ArgumentGreaterOrEqualThan(500, maxBufferSize, "maxBufferSize");
Guard.ArgumentGreaterOrEqualThan(0, bufferingCount, "bufferingCount");
Guard.ArgumentIsValidTimeout(bufferingInterval, "bufferingInterval");
ValidateSqlConnectionString(connectionString, "connectionString");
this.instanceName = instanceName;
this.connectionString = connectionString;
this.tableName = tableName;
this.storedProcedureName = storedProcedureName;
this.onCompletedTimeout = onCompletedTimeout;
retryPolicy.Retrying += Retrying;
var sinkId = I18NHelper.FormatInvariant("BlueprintSqlDatabaseSink ({0})", instanceName);
bufferedPublisher = BufferedEventPublisher <EventEntry> .CreateAndStart(sinkId, PublishEventsAsync, bufferingInterval, bufferingCount, maxBufferSize, cancellationTokenSource.Token);
}
/// <summary>
/// Initializes a new instance of the <see cref="ElasticsearchSink"/> class with the specified connection string and table address.
/// </summary>
/// <param name="instanceName">The name of the instance originating the entries.</param>
/// <param name="connectionString">The connection string for the storage account.</param>
/// <param name="index">Index name prefix formatted as index-{0:yyyy.MM.DD}.</param>
/// <param name="type">Elasticsearch entry type.</param>
/// <param name="flattenPayload">Flatten the payload collection when serializing event entries</param>
/// <param name="bufferInterval">The buffering interval to wait for events to accumulate before sending them to Elasticsearch.</param>
/// <param name="bufferingCount">The buffering event entry count to wait before sending events to Elasticsearch </param>
/// <param name="maxBufferSize">The maximum number of entries that can be buffered while it's sending to Windows Azure Storage before the sink starts dropping entries.</param>
/// <param name="userName">The username to authenticate with Elasticsearch using Basic HTTP authentication.</param>
/// <param name="password">The password to authenticate with Elasticsearch using Basic HTTP authentication.</param>
/// <param name="onCompletedTimeout">Defines a timeout interval for when flushing the entries after an <see cref="OnCompleted"/> call is received and before disposing the sink.
/// This means that if the timeout period elapses, some event entries will be dropped and not sent to the store. Normally, calling <see cref="IDisposable.Dispose"/> on
/// the <see cref="System.Diagnostics.Tracing.EventListener"/> will block until all the entries are flushed or the interval elapses.
/// If <see langword="null"/> is specified, then the call will block indefinitely until the flush operation finishes.</param>
/// <param name="jsonGlobalContextExtension">A json encoded key/value set of global environment parameters to be included in each log entry</param>
public ElasticsearchSink(string instanceName, string connectionString, string index, string type, bool?flattenPayload, TimeSpan bufferInterval,
int bufferingCount, int maxBufferSize, TimeSpan onCompletedTimeout, string userName, string password, string jsonGlobalContextExtension = null)
{
Guard.ArgumentNotNullOrEmpty(instanceName, "instanceName");
Guard.ArgumentNotNullOrEmpty(connectionString, "connectionString");
Guard.ArgumentNotNullOrEmpty(index, "index");
Guard.ArgumentNotNullOrEmpty(type, "type");
Guard.ArgumentIsValidTimeout(onCompletedTimeout, "onCompletedTimeout");
Guard.ArgumentGreaterOrEqualThan(0, bufferingCount, "bufferingCount");
if (Regex.IsMatch(index, "[\\\\/*?\",<>|\\sA-Z]"))
{
throw new ArgumentException(Resource.InvalidElasticsearchIndexNameError, "index");
}
this.onCompletedTimeout = onCompletedTimeout;
this.instanceName = instanceName;
this.flattenPayload = flattenPayload ?? true;
this.elasticsearchUrl = new Uri(new Uri(connectionString), BulkServiceOperationPath);
this.index = index;
this.type = type;
if (!string.IsNullOrEmpty(userName) && !string.IsNullOrEmpty(password))
{
var headerValue = Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Join(":", userName, password)));
this.client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", headerValue);
}
var sinkId = string.Format(CultureInfo.InvariantCulture, "ElasticsearchSink ({0})", instanceName);
bufferedPublisher = BufferedEventPublisher <EventEntry> .CreateAndStart(sinkId, PublishEventsAsync, bufferInterval,
bufferingCount, maxBufferSize, cancellationTokenSource.Token);
this._jsonGlobalContextExtension = !string.IsNullOrEmpty(jsonGlobalContextExtension)? JsonConvert.DeserializeObject <Dictionary <string, string> >(jsonGlobalContextExtension): null;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
}
public void when_no_interval_and_no_count_throw()
{
BufferedEventPublisher <int> .CreateAndStart("sink", b => { return(Task.FromResult(b.Count)); }, Timeout.InfiniteTimeSpan, 0, 1000, CancellationToken.None);
}
protected override void Given()
{
this.sut = BufferedEventPublisher<int>.CreateAndStart("sink", PublishEventsAsync, bufferingInterval, 0, 1000, new CancellationToken());
}
public void when_negative_bufferingCount_throw()
{
BufferedEventPublisher <int> .CreateAndStart("sink", b => { return(Task.FromResult(b.Count)); }, TimeSpan.FromSeconds(5), -1, 1000, CancellationToken.None);
}
protected override void Given()
{
this.Sut = new BufferedEventPublisher<int>("sink", PublishEventsAsync, bufferingInterval, 0, 1000, new CancellationToken());
}
public void when_zero_interval_is_set_default_minimum()
{
var sut = BufferedEventPublisher <int> .CreateAndStart("sink", b => { return(Task.FromResult(b.Count)); }, TimeSpan.Zero, 0, 1000, CancellationToken.None);
Assert.IsNotNull(sut);
}
public void when_null_eventPublisherAction_throws()
{
BufferedEventPublisher <int> .CreateAndStart("sink", null, TimeSpan.Zero, 1, 1000, CancellationToken.None);
}
/// <summary>
/// Create sink
/// </summary>
/// <param name="projectId">Project id</param>
/// <param name="datasetId">Dataset id</param>
/// <param name="tableId">Table id. Expand through DateTime.Format(). e.g. "accesslogyyyyMMdd" => accesslog20150101 <see cref="https://msdn.microsoft.com/en-us/library/vstudio/zdtaw1bw(v=vs.100).aspx"/></param>
/// <param name="authMethod">private_key</param>
/// <param name="serviceAccountEmail">000000000000-xxxxxxxxxxxxxxxxxxxxxx@developer.gserviceaccount.com</param>
/// <param name="privateKeyFile">/path/to/xxxx-000000000000.p12</param>
/// <param name="privateKeyPassphrase">notasecret</param>
/// <param name="autoCreateTable">Create table if it does not exists</param>
/// <param name="tableSchemaFile">Json file path that bigquery table schema</param>
/// <param name="insertIdFieldName">The field name of InsertId</param>
/// <param name="bufferingInterval"></param>
/// <param name="bufferingCount"></param>
/// <param name="bufferingFlushAllTimeout"></param>
/// <param name="maxBufferSize"></param>
public BigQuerySink(
string projectId,
string datasetId,
string tableId,
string authMethod = null,
string serviceAccountEmail = null,
string privateKeyFile = null,
string privateKeyPassphrase = null,
bool?autoCreateTable = null,
string tableSchemaFile = null,
string insertIdFieldName = null,
TimeSpan?bufferingInterval = null,
int?bufferingCount = null,
TimeSpan?bufferingFlushAllTimeout = null,
int?maxBufferSize = null)
{
if (authMethod == null)
{
authMethod = "private_key";
}
if (authMethod != "private_key")
{
throw new NotSupportedException("authMethod must be 'private_key'");
}
if (String.IsNullOrEmpty(serviceAccountEmail))
{
throw new ArgumentException("serviceAccountEmail");
}
if (String.IsNullOrEmpty(privateKeyFile))
{
throw new ArgumentException("privateKeyFile");
}
if (privateKeyPassphrase == null)
{
privateKeyPassphrase = "notasecret";
}
if (String.IsNullOrEmpty(projectId))
{
throw new ArgumentException("projectId");
}
if (String.IsNullOrEmpty(datasetId))
{
throw new ArgumentException("datasetId");
}
if (String.IsNullOrEmpty(tableId))
{
throw new ArgumentException("tableId");
}
if (bufferingInterval == null)
{
bufferingInterval = Constants.DefaultBufferingInterval;
}
if (bufferingCount == null)
{
bufferingCount = Constants.DefaultBufferingCount;
}
if (bufferingFlushAllTimeout == null)
{
bufferingFlushAllTimeout = Constants.DefaultBufferingFlushAllTimeout;
}
if (maxBufferSize == null)
{
maxBufferSize = Constants.DefaultMaxBufferSize;
}
this.ProjectId = projectId;
this.DatasetId = datasetId;
this.TableId = tableId;
this.AutoCreateTable = autoCreateTable ?? false;
this.TableSchemaFile = tableSchemaFile;
this.InsertIdFieldName = insertIdFieldName;
// Load table schema file
if (TableSchemaFile != null)
{
TableSchema = LoadTableSchema(TableSchemaFile);
}
if (TableSchema == null)
{
throw new Exception("table schema not set");
}
// Expand table id 1st time within force mode
ExpandTableIdIfNecessary(force: true);
// configure finished
BigQuerySinkEventSource.Log.SinkStarted("TableId: " + TableIdExpanded);
// Setup bigquery client
var certificate = new X509Certificate2(
privateKeyFile,
privateKeyPassphrase,
X509KeyStorageFlags.MachineKeySet | X509KeyStorageFlags.Exportable);
ServiceAccountCredential credential = new ServiceAccountCredential(
new ServiceAccountCredential.Initializer(serviceAccountEmail)
{
Scopes = new[] { BigqueryService.Scope.Bigquery,
BigqueryService.Scope.BigqueryInsertdata,
BigqueryService.Scope.CloudPlatform,
BigqueryService.Scope.DevstorageFullControl }
}.FromCertificate(certificate));
_BQSvc = new BigqueryService(new BaseClientService.Initializer
{
HttpClientInitializer = credential,
});
_BackOff = new ExponentialBackOff();
_bufferingFlushAllTimeout = bufferingFlushAllTimeout.Value;
_bufferedPublisher = BufferedEventPublisher <EventEntry> .CreateAndStart(
"BigQuery",
PublishEventsAsync,
bufferingInterval.Value,
bufferingCount.Value,
maxBufferSize.Value,
_cancellationTokenSource.Token);
}
public void when_zero_interval_is_set_default_minimum()
{
var sut = new BufferedEventPublisher<int>("sink", b => { return Task.FromResult(b.Count); }, TimeSpan.Zero, 0, 1000, CancellationToken.None);
Assert.IsNotNull(sut);
}
public void when_maxBufferSize_smaller_than_3times_count_throw()
{
BufferedEventPublisher <int> .CreateAndStart("sink", b => { return(Task.FromResult(b.Count)); }, Timeout.InfiniteTimeSpan, 1000, 2999, CancellationToken.None);
}
protected override void Given()
{
this.totalEventsToPublish = 2;
this.sut = BufferedEventPublisher <int> .CreateAndStart("sink", PublishEventsAsync, Timeout.InfiniteTimeSpan, this.totalEventsToPublish, 1000, new CancellationToken());
}
public void when_overrange_maxbuffering_throws()
{
BufferedEventPublisher <int> .CreateAndStart("sink", b => { return(Task.FromResult(b.Count)); }, Timeout.InfiniteTimeSpan, 1, 499, CancellationToken.None);
}
public void when_overrange_bufferingInterval_throws()
{
BufferedEventPublisher <int> .CreateAndStart("sink", b => { return(Task.FromResult(b.Count)); }, TimeSpan.FromMilliseconds(Convert.ToInt64(int.MaxValue) + 1L), 1, 1000, CancellationToken.None);
}
protected override void Given()
{
this.sut = BufferedEventPublisher <int> .CreateAndStart("sink", PublishEventsAsync, bufferingInterval, 0, 1000, new CancellationToken());
}
protected override void Given()
{
this.totalEventsToPublish = 2;
this.sut = BufferedEventPublisher <int> .CreateAndStart("sink", PublishEventsAsync, TimeSpan.FromMilliseconds(500), this.totalEventsToPublish + 1, 1000, new CancellationToken());
}
public void when_null_sinkId_throws()
{
BufferedEventPublisher <int> .CreateAndStart(null, b => { return(Task.FromResult(b.Count)); }, TimeSpan.Zero, 1, 1000, CancellationToken.None);
}
protected override void Given()
{
this.Sut = new BufferedEventPublisher <int>("sink", PublishEventsAsync, bufferingInterval, 0, 1000, new CancellationToken());
}
