internal static KafkaFailoverSink Create(KafkaSink kafkaSink, ILogEventSink fallbackSink,
BatchOptions batchOptions, TimeSpan fallback) =>
batchOptions.QueueLimit.HasValue
? new KafkaFailoverSink(kafkaSink, fallbackSink, batchOptions.BatchSizeLimit, batchOptions.Period,
batchOptions.QueueLimit.Value, fallback)
: new KafkaFailoverSink(kafkaSink, fallbackSink, batchOptions.BatchSizeLimit, batchOptions.Period,
fallback);
internal static KafkaFailoverSink Create(KafkaSink kafkaSink, ILogEventSink fallbackSink,
BatchOptions batchOptions, IModeSwitcher modeSwitcher) =>
batchOptions.QueueLimit.HasValue
? new KafkaFailoverSink(kafkaSink, fallbackSink, batchOptions.BatchSizeLimit, batchOptions.Period,
batchOptions.QueueLimit.Value, modeSwitcher)
: new KafkaFailoverSink(kafkaSink, fallbackSink, batchOptions.BatchSizeLimit, batchOptions.Period,
modeSwitcher);
private void LoadBatchOptions()
{
try
{
string appDirectory = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "Solibri Batch Manager");
if (!Directory.Exists(appDirectory))
{
Directory.CreateDirectory(appDirectory);
}
batchOptionFile = System.IO.Path.Combine(appDirectory, batchOptionFile);
if (!File.Exists(batchOptionFile))
{
BatchOptions options = new BatchOptions();
options.WriteDefault();
SettingUtils.WriteSettings(batchOptionFile, options);
settings.Options = options;
}
else
{
settings.Options = SettingUtils.ReadSettings(batchOptionFile);
}
}
catch (Exception ex)
{
string message = ex.Message;
}
}
public static void Configure(CommandLineApplication command)
{
command.Description = "Adds a dataset as an uploaded nomination (for refreshing catalog)";
command.SetDefaultHelp();
var datasetIdOpt = command.Option("--datasetId | -dsid <datasetId>", "The dataset identifier", CommandOptionType.SingleValue);
var queueOpt = command.Option("--queue | -q", "Queues the job to catalog the dataset.", CommandOptionType.NoValue);
command.OnExecute(async() =>
{
var cosmos = new CosmosOptions();
var contact = new ContactInfoOptions();
var batch = new BatchOptions();
var datasetId = datasetIdOpt.Value();
if (command.HasAllRequiredParameters(new[]
{
cosmos.Endpoint,
cosmos.Database,
contact.Name,
contact.Email,
batch.Url,
batch.Account,
batch.Key,
datasetId,
}))
{
await new Admin.Dataset.DatasetNominateTask(cosmos, contact, batch, datasetId, queueOpt.HasValue()).ExecuteAsync();
}
return(0);
});
}
public async Task CreateBatchAsyncRespectsTheMaximumSizeWhenProvided()
{
var expectedMaximumSize = 512;
var options = new BatchOptions {
MaximumSizeInBytes = expectedMaximumSize
};
var retryPolicy = new BasicRetryPolicy(new RetryOptions {
TryTimeout = TimeSpan.FromSeconds(17)
});
var producer = new Mock <AmqpProducer>("aHub", null, Mock.Of <AmqpConnectionScope>(), new AmqpMessageConverter(), retryPolicy)
{
CallBase = true
};
producer
.Protected()
.Setup <Task <SendingAmqpLink> >("CreateLinkAndEnsureProducerStateAsync",
ItExpr.IsAny <string>(),
ItExpr.IsAny <TimeSpan>(),
ItExpr.IsAny <CancellationToken>())
.Callback(() => SetMaximumMessageSize(producer.Object, expectedMaximumSize + 27))
.Returns(Task.FromResult(new SendingAmqpLink(new AmqpLinkSettings())))
.Verifiable();
using TransportEventBatch batch = await producer.Object.CreateBatchAsync(options, default);
Assert.That(options.MaximumSizeInBytes, Is.EqualTo(expectedMaximumSize));
}
public void TryAddSetsTheCount()
{
var currentIndex = -1;
var options = new BatchOptions {
MaximumizeInBytes = 5000
};
var eventMessages = new AmqpMessage[5];
var mockEnvelope = new Mock <AmqpMessage>();
var mockConverter = new InjectableMockConverter();
mockConverter.CreateBatchFromEventsHandler = (_e, _p) => mockEnvelope.Object;
mockConverter.CreateMessageFromEventHandler = (_e, _p) => eventMessages[++currentIndex];
mockEnvelope
.Setup(message => message.SerializedMessageSize)
.Returns(0);
for (var index = 0; index < eventMessages.Length; ++index)
{
eventMessages[index] = AmqpMessage.Create(new Data {
Value = new ArraySegment <byte>(new byte[] { 0x66 })
});
}
// Add the messages to the batch; all should be accepted.
var batch = new AmqpEventBatch(mockConverter, options);
for (var index = 0; index < eventMessages.Length; ++index)
{
Assert.That(batch.TryAdd(new EventData(new byte[0])), Is.True, $"The addition for index: { index } should fit and be accepted.");
}
Assert.That(batch.Count, Is.EqualTo(eventMessages.Length), "The count should have been set when the batch was updated.");
}
public void CreateBatchAsyncVerifiesTheMaximumSize()
{
var linkMaximumSize = 512;
var options = new BatchOptions {
MaximumSizeInBytes = 1024
};
var retryPolicy = new BasicRetryPolicy(new RetryOptions {
TryTimeout = TimeSpan.FromSeconds(17)
});
var producer = new Mock <AmqpProducer>("aHub", null, Mock.Of <AmqpConnectionScope>(), new AmqpMessageConverter(), retryPolicy)
{
CallBase = true
};
producer
.Protected()
.Setup <Task <SendingAmqpLink> >("CreateLinkAndEnsureProducerStateAsync",
ItExpr.IsAny <string>(),
ItExpr.IsAny <TimeSpan>(),
ItExpr.IsAny <CancellationToken>())
.Callback(() => SetMaximumMessageSize(producer.Object, linkMaximumSize))
.Returns(Task.FromResult(new SendingAmqpLink(new AmqpLinkSettings())))
.Verifiable();
Assert.That(async() => await producer.Object.CreateBatchAsync(options, default), Throws.InstanceOf <ArgumentOutOfRangeException>());
}
public async Task SendEnumerableUsesThePartitionKey()
{
var expectedPartitionKey = "some key";
var options = new BatchOptions {
PartitionKey = expectedPartitionKey
};
var retryPolicy = new BasicRetryPolicy(new RetryOptions {
TryTimeout = TimeSpan.FromSeconds(17)
});
var producer = new Mock <AmqpProducer>("aHub", null, Mock.Of <AmqpConnectionScope>(), new AmqpMessageConverter(), retryPolicy)
{
CallBase = true
};
producer
.Protected()
.Setup <Task>("SendAsync",
ItExpr.IsAny <Func <AmqpMessage> >(),
ItExpr.Is <string>(value => value == expectedPartitionKey),
ItExpr.IsAny <CancellationToken>())
.Returns(Task.CompletedTask)
.Verifiable();
await producer.Object.SendAsync(new[] { new EventData(new byte[] { 0x15 }) }, options, CancellationToken.None);
producer.VerifyAll();
}
public async Task CreateBatchAsyncBuildsAnAmqpEventBatchWithTheOptions()
{
var options = new BatchOptions {
MaximumSizeInBytes = 512
};
var retryPolicy = new BasicRetryPolicy(new RetryOptions {
TryTimeout = TimeSpan.FromSeconds(17)
});
var producer = new Mock <AmqpProducer>("aHub", null, Mock.Of <AmqpConnectionScope>(), new AmqpMessageConverter(), retryPolicy)
{
CallBase = true
};
producer
.Protected()
.Setup <Task <SendingAmqpLink> >("CreateLinkAndEnsureProducerStateAsync",
ItExpr.IsAny <string>(),
ItExpr.IsAny <TimeSpan>(),
ItExpr.IsAny <CancellationToken>())
.Callback(() => SetMaximumMessageSize(producer.Object, options.MaximumSizeInBytes.Value + 982))
.Returns(Task.FromResult(new SendingAmqpLink(new AmqpLinkSettings())))
.Verifiable();
using TransportEventBatch batch = await producer.Object.CreateBatchAsync(options, default);
Assert.That(batch, Is.Not.Null, "The created batch should be populated.");
Assert.That(batch, Is.InstanceOf <AmqpEventBatch>(), $"The created batch should be an { nameof(AmqpEventBatch) }.");
Assert.That(GetEventBatchOptions((AmqpEventBatch)batch), Is.SameAs(options), "The provided options should have been used.");
}
public async Task SendingPartitionKeyBatchOnPartitionSenderShouldFail()
{
var ehClient = EventHubClient.CreateFromConnectionString(TestUtility.EventHubsConnectionString);
var partitionSender = ehClient.CreatePartitionSender("0");
try
{
var batchOptions = new BatchOptions()
{
PartitionKey = "this is the partition key"
};
var batcher = ehClient.CreateBatch(batchOptions);
await Assert.ThrowsAsync <InvalidOperationException>(async() =>
{
TestUtility.Log("Attempting to send a partition-key batch on partition sender. This should fail.");
await partitionSender.SendAsync(batcher);
});
}
finally
{
await Task.WhenAll(
partitionSender.CloseAsync(),
ehClient.CloseAsync());
}
}
public static BatchOptions ReadSettings(string xmlFile)
{
BatchOptions batchOptions = new BatchOptions();
try
{
if (File.Exists(xmlFile))
{
XmlSerializer serializer = new XmlSerializer(typeof(BatchOptions));
using (FileStream fs = new FileStream(xmlFile, FileMode.Open))
{
XmlReader reader = XmlReader.Create(fs);
if (serializer.CanDeserialize(reader))
{
batchOptions = (BatchOptions)serializer.Deserialize(reader);
}
fs.Close();
}
}
}
catch (Exception ex)
{
string message = ex.Message;
}
return(batchOptions);
}
public void TryAddDoesNotAcceptAnEventBiggerThanTheMaximumSize()
{
var maximumSize = 50;
var batchEnvelopeSize = 0;
var options = new BatchOptions {
MaximumizeInBytes = maximumSize
};
var mockEnvelope = new Mock <AmqpMessage>();
var mockEvent = new Mock <AmqpMessage>();
var mockConverter = new InjectableMockConverter();
mockConverter.CreateBatchFromEventsHandler = (_e, _p) => mockEnvelope.Object;
mockConverter.CreateMessageFromEventHandler = (_e, _p) => mockEvent.Object;
mockEnvelope
.Setup(message => message.SerializedMessageSize)
.Returns(batchEnvelopeSize);
mockEvent
.Setup(message => message.SerializedMessageSize)
.Returns(maximumSize);
var batch = new AmqpEventBatch(mockConverter, options);
Assert.That(batch.TryAdd(new EventData(new byte[0])), Is.False, "An event of the maximum size is too large due to the reserved overhead.");
}
public void TryAddAcceptsAnEventSmallerThanTheMaximumSize()
{
var maximumSize = 50;
var eventMessageSize = 40;
var options = new BatchOptions {
MaximumizeInBytes = maximumSize
};
var mockEnvelope = new Mock <AmqpMessage>();
var mockEvent = new Mock <AmqpMessage>();
var mockConverter = new InjectableMockConverter();
mockConverter.CreateBatchFromEventsHandler = (_e, _p) => mockEnvelope.Object;
mockConverter.CreateMessageFromEventHandler = (_e, _p) => mockEvent.Object;
mockEnvelope
.Setup(message => message.SerializedMessageSize)
.Returns(0);
mockEvent
.Setup(message => message.SerializedMessageSize)
.Returns(eventMessageSize);
var batch = new AmqpEventBatch(mockConverter, options);
Assert.That(batch.TryAdd(new EventData(new byte[0])), Is.True);
}
public async Task SendBatchRespectsTheCancellationTokenIfSetWhenCalled()
{
var expectedMaximumSize = 512;
var options = new BatchOptions();
var retryPolicy = new BasicRetryPolicy(new RetryOptions {
TryTimeout = TimeSpan.FromSeconds(17)
});
var producer = new Mock <AmqpProducer>("aHub", null, Mock.Of <AmqpConnectionScope>(), new AmqpMessageConverter(), retryPolicy)
{
CallBase = true
};
producer
.Protected()
.Setup <Task <SendingAmqpLink> >("CreateLinkAndEnsureProducerStateAsync",
ItExpr.IsAny <string>(),
ItExpr.IsAny <TimeSpan>(),
ItExpr.IsAny <CancellationToken>())
.Callback(() => SetMaximumMessageSize(producer.Object, expectedMaximumSize))
.Returns(Task.FromResult(new SendingAmqpLink(new AmqpLinkSettings())));
using TransportEventBatch batch = await producer.Object.CreateBatchAsync(options, default);
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.Cancel();
Assert.That(async() => await producer.Object.SendAsync(new EventDataBatch(batch, options), cancellationSource.Token), Throws.InstanceOf <TaskCanceledException>());
}
public async Task ProducerCanSendAnEventBatchUsingAPartitionHashKey()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
{
IEnumerable <EventData> events = Enumerable
.Range(0, 25)
.Select(index => new EventData(Encoding.UTF8.GetBytes(new string('X', index + 5))));
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var batchOptions = new BatchOptions {
PartitionKey = "some123key-!d"
};
await using (var client = new EventHubClient(connectionString))
await using (EventHubProducer producer = client.CreateProducer())
{
using EventDataBatch batch = await producer.CreateBatchAsync(batchOptions);
foreach (EventData eventData in events)
{
Assert.That(() => batch.TryAdd(eventData), Is.True, "An event was rejected by the batch; all events should be accepted.");
}
Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing);
}
}
}
public void SendBatchRespectsTheRetryPolicy(RetryOptions retryOptions)
{
var partitionKey = "testMe";
var options = new BatchOptions {
PartitionKey = partitionKey
};
var retriableException = new EventHubsException(true, "Test");
var retryPolicy = new BasicRetryPolicy(retryOptions);
var batch = new EventDataBatch(Mock.Of <TransportEventBatch>(), options);
var producer = new Mock <AmqpProducer>("aHub", null, Mock.Of <AmqpConnectionScope>(), new AmqpMessageConverter(), retryPolicy)
{
CallBase = true
};
producer
.Protected()
.Setup <Task <SendingAmqpLink> >("CreateLinkAndEnsureProducerStateAsync",
ItExpr.IsAny <string>(),
ItExpr.IsAny <TimeSpan>(),
ItExpr.IsAny <CancellationToken>())
.Throws(retriableException);
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
Assert.That(async() => await producer.Object.SendAsync(batch, cancellationSource.Token), Throws.InstanceOf(retriableException.GetType()));
producer
.Protected()
.Verify("CreateLinkAndEnsureProducerStateAsync", Times.Exactly(1 + retryOptions.MaximumRetries),
ItExpr.Is <string>(value => value == null),
ItExpr.IsAny <TimeSpan>(),
ItExpr.IsAny <CancellationToken>());
}
public async Task SendingPartitionKeyBatchOnPartitionSenderShouldFail()
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestUtility.BuildEventHubsConnectionString(scope.EventHubName);
var ehClient = EventHubClient.CreateFromConnectionString(connectionString);
var partitionSender = default(PartitionSender);
try
{
var partitions = await GetPartitionsAsync(ehClient);
var partitionId = partitions[this.random.Next(partitions.Length)];
partitionSender = ehClient.CreatePartitionSender(partitionId);
var batchOptions = new BatchOptions()
{
PartitionKey = "this is the partition key"
};
var batcher = ehClient.CreateBatch(batchOptions);
await Assert.ThrowsAsync <InvalidOperationException>(async() =>
{
TestUtility.Log("Attempting to send a partition-key batch on partition sender. This should fail.");
await partitionSender.SendAsync(batcher);
});
}
finally
{
await Task.WhenAll(
partitionSender.CloseAsync(),
ehClient.CloseAsync());
}
}
}
protected override int LoadInternal(IEnumerable <ICommandData> items, DocumentsOperationContext context, EtlStatsScope scope)
{
var commands = items as List <ICommandData>;
Debug.Assert(commands != null);
if (commands.Count == 0)
{
return(0);
}
if (ShouldTrackTimeSeries())
{
foreach (var command in commands)
{
if (command is TimeSeriesBatchCommandData tsbc)
{
if (TimeSeriesHandler.CheckIfIncrementalTs(tsbc.Name))
{
throw new NotSupportedException($"Load isn't support for incremental time series '{tsbc.Name}' at document '{tsbc.Id}'");
}
}
}
}
BatchOptions options = null;
if (Configuration.LoadRequestTimeoutInSec != null)
{
options = new BatchOptions
{
RequestTimeout = TimeSpan.FromSeconds(Configuration.LoadRequestTimeoutInSec.Value)
};
}
using (var batchCommand = new SingleNodeBatchCommand(DocumentConventions.DefaultForServer, context, commands, options))
{
var duration = Stopwatch.StartNew();
try
{
BeforeActualLoad?.Invoke(this);
AsyncHelpers.RunSync(() => _requestExecutor.ExecuteAsync(batchCommand, context, token: CancellationToken));
_recentUrl = _requestExecutor.Url;
return(commands.Count);
}
catch (OperationCanceledException e)
{
if (CancellationToken.IsCancellationRequested == false)
{
ThrowTimeoutException(commands.Count, duration.Elapsed, e);
}
throw;
}
}
}
public void QueueLimitSetter_ShouldThrowsException_WhenValueIsNonPositive(int value)
{
// Arrange
var options = new BatchOptions();
// Act + Assert
Assert.Throws <ArgumentOutOfRangeException>(() => options.QueueLimit = value);
}
private void __bw_DoWork(object sender, DoWorkEventArgs e)
{
BatchOptions options = e.Argument as BatchOptions;
_process.MajorProgressChanged += new EventHandler(process_MajorProgressChanged);
_process.MinorProgressChanged += new EventHandler(process_MinorProgressChanged);
_process.Process(options);
}
public void PeriodSetter_ShouldThrowsException_WhenValueIsNegative(TimeSpan value)
{
// Arrange
var options = new BatchOptions();
// Act + Assert
Assert.Throws <ArgumentOutOfRangeException>(() => options.Period = value);
}
public void execute(BatchOptions options, string filename)
{
//todo calc new filename
string newFilename = filename;
//rename by copy-deleting
File.Copy(filename, newFilename);
File.Delete(filename);
}
public async Task MultiBatchOperationInsertTest()
{
var inputbatch = new InputMultiBatchOperation
{
BatchQueries = new BatchOperationQuery[] {
new BatchOperationQuery {
BatchInputQuery = @"delete from batch_table_test", InputJson = ""
},
new BatchOperationQuery {
BatchInputQuery = @"insert into batch_table_test (NR,NAM)values(:NR,:NAM)", InputJson = "[{\"NR\": 111, \"NAM\":\"nannaa1\"},{\"NR\":222, \"NAM\":\"nannaa2\"},{\"NR\":333, \"NAM\":\"nannaa3\"}, {\"NR\":444, \"NAM\":\"nannaa4\"}]"
},
new BatchOperationQuery {
BatchInputQuery = @"insert into batch_table_test (NR,NAM)values(:NR,:NAM)", InputJson = "[{\"NR\": 555, \"NAM\":\"nannaa1\"},{\"NR\":666, \"NAM\":\"nannaa2\"}]"
}
},
ConnectionString = ConnectionString
};
var options = new BatchOptions
{
ThrowErrorOnFailure = true,
IsolationLevel = Oracle_IsolationLevel.Serializable
};
MultiBatchOperationOutput output;
try
{
output = await OracleTasks.MultiBatchOperationOracle(inputbatch, options, new CancellationToken());
}
catch (Exception ee)
{
throw ee;
}
var o = new QueryOutputProperties
{
ReturnType = QueryReturnType.Json,
JsonOutput = new JsonOutputProperties(),
OutputToFile = false
};
var q2 = new QueryProperties {
Query = @"select count(*) as ROWCOUNT from batch_table_test", ConnectionString = ConnectionString
};
var options_2 = new QueryOptions();
options.ThrowErrorOnFailure = true;
options.IsolationLevel = Oracle_IsolationLevel.Serializable;
var result_debug = await OracleTasks.ExecuteQueryOracle(q2, o, options_2, new CancellationToken());
Assert.AreEqual(result_debug.Result, "[\r\n {\r\n \"ROWCOUNT\": 6.0\r\n }\r\n]");
}
public void QueueLimitSetter_ShouldNotThrowsException_WhenValueIsPositive(int?value)
{
// Arrange
var options = new BatchOptions();
// Act
options.QueueLimit = value;
// Assert
Assert.Equal(value, options.QueueLimit);
}
public void PeriodSetter_ShouldNotThrowsException_WhenValueIsNotNegative(TimeSpan value)
{
// Arrange
var options = new BatchOptions();
// Act
options.Period = value;
// Assert
Assert.Equal(value, options.Period);
}
public void SendAllowsAPartitionHashKeyWithABatch()
{
var batchOptions = new BatchOptions {
PartitionKey = "testKey"
};
var batch = new EventDataBatch(new MockTransportBatch(), batchOptions);
var transportProducer = new ObservableTransportProducerMock();
var producer = new EventHubProducerClient(new MockConnection(transportProducer));
Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing);
}
public void SendAllowsAPartitionHashKeyWithABatch()
{
var batchOptions = new BatchOptions {
PartitionKey = "testKey"
};
var batch = new EventDataBatch(new MockTransportBatch(), batchOptions);
var transportProducer = new ObservableTransportProducerMock();
var producer = new EventHubProducer(transportProducer, new Uri("amqp://some.endpoint.com/path"), "dummy", new EventHubProducerOptions(), Mock.Of <EventHubRetryPolicy>());
Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing);
}
public async Task CreateBatchSetsTheSendOptionsForTheEventBatch()
{
var batchOptions = new BatchOptions {
PartitionKey = "Hi", MaximumSizeInBytes = 9999
};
var transportProducer = new ObservableTransportProducerMock();
var producer = new EventHubProducerClient(new MockConnection(transportProducer));
var eventBatch = await producer.CreateBatchAsync(batchOptions);
Assert.That(eventBatch.SendOptions, Is.SameAs(transportProducer.CreateBatchCalledWith), "The batch options should have used for the send options.");
;
}
public void CreateBatchForASpecificPartitionDoesNotAllowAPartitionHashKey()
{
var batchOptions = new BatchOptions {
PartitionKey = "testKey"
};
var transportProducer = new ObservableTransportProducerMock();
var producer = new EventHubProducer(transportProducer, new Uri("amqp://some.endpoint.com/path"), "dummy", new EventHubProducerOptions {
PartitionId = "1"
}, Mock.Of <EventHubRetryPolicy>());
Assert.That(async() => await producer.CreateBatchAsync(batchOptions), Throws.InvalidOperationException);
}
public async Task CreateBatchSetsTheSendOptionsForTheEventBatch()
{
var batchOptions = new BatchOptions {
PartitionKey = "Hi", MaximumizeInBytes = 9999
};
var transportProducer = new ObservableTransportProducerMock();
var producer = new EventHubProducer(transportProducer, new Uri("amqp://some.endpoint.com/path"), "dummy", new EventHubProducerOptions(), Mock.Of <EventHubRetryPolicy>());
EventDataBatch eventBatch = await producer.CreateBatchAsync(batchOptions);
Assert.That(eventBatch.SendOptions, Is.SameAs(transportProducer.CreateBatchCalledWith), "The batch options should have used for the send options.");
;
}
