internal static KafkaFailoverSink Create(KafkaSink kafkaSink, ILogEventSink fallbackSink, BatchOptions batchOptions, TimeSpan fallback) => batchOptions.QueueLimit.HasValue ? new KafkaFailoverSink(kafkaSink, fallbackSink, batchOptions.BatchSizeLimit, batchOptions.Period, batchOptions.QueueLimit.Value, fallback) : new KafkaFailoverSink(kafkaSink, fallbackSink, batchOptions.BatchSizeLimit, batchOptions.Period, fallback);
internal static KafkaFailoverSink Create(KafkaSink kafkaSink, ILogEventSink fallbackSink, BatchOptions batchOptions, IModeSwitcher modeSwitcher) => batchOptions.QueueLimit.HasValue ? new KafkaFailoverSink(kafkaSink, fallbackSink, batchOptions.BatchSizeLimit, batchOptions.Period, batchOptions.QueueLimit.Value, modeSwitcher) : new KafkaFailoverSink(kafkaSink, fallbackSink, batchOptions.BatchSizeLimit, batchOptions.Period, modeSwitcher);
private void LoadBatchOptions() { try { string appDirectory = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "Solibri Batch Manager"); if (!Directory.Exists(appDirectory)) { Directory.CreateDirectory(appDirectory); } batchOptionFile = System.IO.Path.Combine(appDirectory, batchOptionFile); if (!File.Exists(batchOptionFile)) { BatchOptions options = new BatchOptions(); options.WriteDefault(); SettingUtils.WriteSettings(batchOptionFile, options); settings.Options = options; } else { settings.Options = SettingUtils.ReadSettings(batchOptionFile); } } catch (Exception ex) { string message = ex.Message; } }
public static void Configure(CommandLineApplication command) { command.Description = "Adds a dataset as an uploaded nomination (for refreshing catalog)"; command.SetDefaultHelp(); var datasetIdOpt = command.Option("--datasetId | -dsid <datasetId>", "The dataset identifier", CommandOptionType.SingleValue); var queueOpt = command.Option("--queue | -q", "Queues the job to catalog the dataset.", CommandOptionType.NoValue); command.OnExecute(async() => { var cosmos = new CosmosOptions(); var contact = new ContactInfoOptions(); var batch = new BatchOptions(); var datasetId = datasetIdOpt.Value(); if (command.HasAllRequiredParameters(new[] { cosmos.Endpoint, cosmos.Database, contact.Name, contact.Email, batch.Url, batch.Account, batch.Key, datasetId, })) { await new Admin.Dataset.DatasetNominateTask(cosmos, contact, batch, datasetId, queueOpt.HasValue()).ExecuteAsync(); } return(0); }); }
public async Task CreateBatchAsyncRespectsTheMaximumSizeWhenProvided() { var expectedMaximumSize = 512; var options = new BatchOptions { MaximumSizeInBytes = expectedMaximumSize }; var retryPolicy = new BasicRetryPolicy(new RetryOptions { TryTimeout = TimeSpan.FromSeconds(17) }); var producer = new Mock <AmqpProducer>("aHub", null, Mock.Of <AmqpConnectionScope>(), new AmqpMessageConverter(), retryPolicy) { CallBase = true }; producer .Protected() .Setup <Task <SendingAmqpLink> >("CreateLinkAndEnsureProducerStateAsync", ItExpr.IsAny <string>(), ItExpr.IsAny <TimeSpan>(), ItExpr.IsAny <CancellationToken>()) .Callback(() => SetMaximumMessageSize(producer.Object, expectedMaximumSize + 27)) .Returns(Task.FromResult(new SendingAmqpLink(new AmqpLinkSettings()))) .Verifiable(); using TransportEventBatch batch = await producer.Object.CreateBatchAsync(options, default); Assert.That(options.MaximumSizeInBytes, Is.EqualTo(expectedMaximumSize)); }
public void TryAddSetsTheCount() { var currentIndex = -1; var options = new BatchOptions { MaximumizeInBytes = 5000 }; var eventMessages = new AmqpMessage[5]; var mockEnvelope = new Mock <AmqpMessage>(); var mockConverter = new InjectableMockConverter(); mockConverter.CreateBatchFromEventsHandler = (_e, _p) => mockEnvelope.Object; mockConverter.CreateMessageFromEventHandler = (_e, _p) => eventMessages[++currentIndex]; mockEnvelope .Setup(message => message.SerializedMessageSize) .Returns(0); for (var index = 0; index < eventMessages.Length; ++index) { eventMessages[index] = AmqpMessage.Create(new Data { Value = new ArraySegment <byte>(new byte[] { 0x66 }) }); } // Add the messages to the batch; all should be accepted. var batch = new AmqpEventBatch(mockConverter, options); for (var index = 0; index < eventMessages.Length; ++index) { Assert.That(batch.TryAdd(new EventData(new byte[0])), Is.True, $"The addition for index: { index } should fit and be accepted."); } Assert.That(batch.Count, Is.EqualTo(eventMessages.Length), "The count should have been set when the batch was updated."); }
public void CreateBatchAsyncVerifiesTheMaximumSize() { var linkMaximumSize = 512; var options = new BatchOptions { MaximumSizeInBytes = 1024 }; var retryPolicy = new BasicRetryPolicy(new RetryOptions { TryTimeout = TimeSpan.FromSeconds(17) }); var producer = new Mock <AmqpProducer>("aHub", null, Mock.Of <AmqpConnectionScope>(), new AmqpMessageConverter(), retryPolicy) { CallBase = true }; producer .Protected() .Setup <Task <SendingAmqpLink> >("CreateLinkAndEnsureProducerStateAsync", ItExpr.IsAny <string>(), ItExpr.IsAny <TimeSpan>(), ItExpr.IsAny <CancellationToken>()) .Callback(() => SetMaximumMessageSize(producer.Object, linkMaximumSize)) .Returns(Task.FromResult(new SendingAmqpLink(new AmqpLinkSettings()))) .Verifiable(); Assert.That(async() => await producer.Object.CreateBatchAsync(options, default), Throws.InstanceOf <ArgumentOutOfRangeException>()); }
public async Task SendEnumerableUsesThePartitionKey() { var expectedPartitionKey = "some key"; var options = new BatchOptions { PartitionKey = expectedPartitionKey }; var retryPolicy = new BasicRetryPolicy(new RetryOptions { TryTimeout = TimeSpan.FromSeconds(17) }); var producer = new Mock <AmqpProducer>("aHub", null, Mock.Of <AmqpConnectionScope>(), new AmqpMessageConverter(), retryPolicy) { CallBase = true }; producer .Protected() .Setup <Task>("SendAsync", ItExpr.IsAny <Func <AmqpMessage> >(), ItExpr.Is <string>(value => value == expectedPartitionKey), ItExpr.IsAny <CancellationToken>()) .Returns(Task.CompletedTask) .Verifiable(); await producer.Object.SendAsync(new[] { new EventData(new byte[] { 0x15 }) }, options, CancellationToken.None); producer.VerifyAll(); }
public async Task CreateBatchAsyncBuildsAnAmqpEventBatchWithTheOptions() { var options = new BatchOptions { MaximumSizeInBytes = 512 }; var retryPolicy = new BasicRetryPolicy(new RetryOptions { TryTimeout = TimeSpan.FromSeconds(17) }); var producer = new Mock <AmqpProducer>("aHub", null, Mock.Of <AmqpConnectionScope>(), new AmqpMessageConverter(), retryPolicy) { CallBase = true }; producer .Protected() .Setup <Task <SendingAmqpLink> >("CreateLinkAndEnsureProducerStateAsync", ItExpr.IsAny <string>(), ItExpr.IsAny <TimeSpan>(), ItExpr.IsAny <CancellationToken>()) .Callback(() => SetMaximumMessageSize(producer.Object, options.MaximumSizeInBytes.Value + 982)) .Returns(Task.FromResult(new SendingAmqpLink(new AmqpLinkSettings()))) .Verifiable(); using TransportEventBatch batch = await producer.Object.CreateBatchAsync(options, default); Assert.That(batch, Is.Not.Null, "The created batch should be populated."); Assert.That(batch, Is.InstanceOf <AmqpEventBatch>(), $"The created batch should be an { nameof(AmqpEventBatch) }."); Assert.That(GetEventBatchOptions((AmqpEventBatch)batch), Is.SameAs(options), "The provided options should have been used."); }
public async Task SendingPartitionKeyBatchOnPartitionSenderShouldFail() { var ehClient = EventHubClient.CreateFromConnectionString(TestUtility.EventHubsConnectionString); var partitionSender = ehClient.CreatePartitionSender("0"); try { var batchOptions = new BatchOptions() { PartitionKey = "this is the partition key" }; var batcher = ehClient.CreateBatch(batchOptions); await Assert.ThrowsAsync <InvalidOperationException>(async() => { TestUtility.Log("Attempting to send a partition-key batch on partition sender. This should fail."); await partitionSender.SendAsync(batcher); }); } finally { await Task.WhenAll( partitionSender.CloseAsync(), ehClient.CloseAsync()); } }
public static BatchOptions ReadSettings(string xmlFile) { BatchOptions batchOptions = new BatchOptions(); try { if (File.Exists(xmlFile)) { XmlSerializer serializer = new XmlSerializer(typeof(BatchOptions)); using (FileStream fs = new FileStream(xmlFile, FileMode.Open)) { XmlReader reader = XmlReader.Create(fs); if (serializer.CanDeserialize(reader)) { batchOptions = (BatchOptions)serializer.Deserialize(reader); } fs.Close(); } } } catch (Exception ex) { string message = ex.Message; } return(batchOptions); }
public void TryAddDoesNotAcceptAnEventBiggerThanTheMaximumSize() { var maximumSize = 50; var batchEnvelopeSize = 0; var options = new BatchOptions { MaximumizeInBytes = maximumSize }; var mockEnvelope = new Mock <AmqpMessage>(); var mockEvent = new Mock <AmqpMessage>(); var mockConverter = new InjectableMockConverter(); mockConverter.CreateBatchFromEventsHandler = (_e, _p) => mockEnvelope.Object; mockConverter.CreateMessageFromEventHandler = (_e, _p) => mockEvent.Object; mockEnvelope .Setup(message => message.SerializedMessageSize) .Returns(batchEnvelopeSize); mockEvent .Setup(message => message.SerializedMessageSize) .Returns(maximumSize); var batch = new AmqpEventBatch(mockConverter, options); Assert.That(batch.TryAdd(new EventData(new byte[0])), Is.False, "An event of the maximum size is too large due to the reserved overhead."); }
public void TryAddAcceptsAnEventSmallerThanTheMaximumSize() { var maximumSize = 50; var eventMessageSize = 40; var options = new BatchOptions { MaximumizeInBytes = maximumSize }; var mockEnvelope = new Mock <AmqpMessage>(); var mockEvent = new Mock <AmqpMessage>(); var mockConverter = new InjectableMockConverter(); mockConverter.CreateBatchFromEventsHandler = (_e, _p) => mockEnvelope.Object; mockConverter.CreateMessageFromEventHandler = (_e, _p) => mockEvent.Object; mockEnvelope .Setup(message => message.SerializedMessageSize) .Returns(0); mockEvent .Setup(message => message.SerializedMessageSize) .Returns(eventMessageSize); var batch = new AmqpEventBatch(mockConverter, options); Assert.That(batch.TryAdd(new EventData(new byte[0])), Is.True); }
public async Task SendBatchRespectsTheCancellationTokenIfSetWhenCalled() { var expectedMaximumSize = 512; var options = new BatchOptions(); var retryPolicy = new BasicRetryPolicy(new RetryOptions { TryTimeout = TimeSpan.FromSeconds(17) }); var producer = new Mock <AmqpProducer>("aHub", null, Mock.Of <AmqpConnectionScope>(), new AmqpMessageConverter(), retryPolicy) { CallBase = true }; producer .Protected() .Setup <Task <SendingAmqpLink> >("CreateLinkAndEnsureProducerStateAsync", ItExpr.IsAny <string>(), ItExpr.IsAny <TimeSpan>(), ItExpr.IsAny <CancellationToken>()) .Callback(() => SetMaximumMessageSize(producer.Object, expectedMaximumSize)) .Returns(Task.FromResult(new SendingAmqpLink(new AmqpLinkSettings()))); using TransportEventBatch batch = await producer.Object.CreateBatchAsync(options, default); using CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.Cancel(); Assert.That(async() => await producer.Object.SendAsync(new EventDataBatch(batch, options), cancellationSource.Token), Throws.InstanceOf <TaskCanceledException>()); }
public async Task ProducerCanSendAnEventBatchUsingAPartitionHashKey() { await using (EventHubScope scope = await EventHubScope.CreateAsync(2)) { IEnumerable <EventData> events = Enumerable .Range(0, 25) .Select(index => new EventData(Encoding.UTF8.GetBytes(new string('X', index + 5)))); var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); var batchOptions = new BatchOptions { PartitionKey = "some123key-!d" }; await using (var client = new EventHubClient(connectionString)) await using (EventHubProducer producer = client.CreateProducer()) { using EventDataBatch batch = await producer.CreateBatchAsync(batchOptions); foreach (EventData eventData in events) { Assert.That(() => batch.TryAdd(eventData), Is.True, "An event was rejected by the batch; all events should be accepted."); } Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing); } } }
public void SendBatchRespectsTheRetryPolicy(RetryOptions retryOptions) { var partitionKey = "testMe"; var options = new BatchOptions { PartitionKey = partitionKey }; var retriableException = new EventHubsException(true, "Test"); var retryPolicy = new BasicRetryPolicy(retryOptions); var batch = new EventDataBatch(Mock.Of <TransportEventBatch>(), options); var producer = new Mock <AmqpProducer>("aHub", null, Mock.Of <AmqpConnectionScope>(), new AmqpMessageConverter(), retryPolicy) { CallBase = true }; producer .Protected() .Setup <Task <SendingAmqpLink> >("CreateLinkAndEnsureProducerStateAsync", ItExpr.IsAny <string>(), ItExpr.IsAny <TimeSpan>(), ItExpr.IsAny <CancellationToken>()) .Throws(retriableException); using CancellationTokenSource cancellationSource = new CancellationTokenSource(); Assert.That(async() => await producer.Object.SendAsync(batch, cancellationSource.Token), Throws.InstanceOf(retriableException.GetType())); producer .Protected() .Verify("CreateLinkAndEnsureProducerStateAsync", Times.Exactly(1 + retryOptions.MaximumRetries), ItExpr.Is <string>(value => value == null), ItExpr.IsAny <TimeSpan>(), ItExpr.IsAny <CancellationToken>()); }
public async Task SendingPartitionKeyBatchOnPartitionSenderShouldFail() { await using (var scope = await EventHubScope.CreateAsync(1)) { var connectionString = TestUtility.BuildEventHubsConnectionString(scope.EventHubName); var ehClient = EventHubClient.CreateFromConnectionString(connectionString); var partitionSender = default(PartitionSender); try { var partitions = await GetPartitionsAsync(ehClient); var partitionId = partitions[this.random.Next(partitions.Length)]; partitionSender = ehClient.CreatePartitionSender(partitionId); var batchOptions = new BatchOptions() { PartitionKey = "this is the partition key" }; var batcher = ehClient.CreateBatch(batchOptions); await Assert.ThrowsAsync <InvalidOperationException>(async() => { TestUtility.Log("Attempting to send a partition-key batch on partition sender. This should fail."); await partitionSender.SendAsync(batcher); }); } finally { await Task.WhenAll( partitionSender.CloseAsync(), ehClient.CloseAsync()); } } }
protected override int LoadInternal(IEnumerable <ICommandData> items, DocumentsOperationContext context, EtlStatsScope scope) { var commands = items as List <ICommandData>; Debug.Assert(commands != null); if (commands.Count == 0) { return(0); } if (ShouldTrackTimeSeries()) { foreach (var command in commands) { if (command is TimeSeriesBatchCommandData tsbc) { if (TimeSeriesHandler.CheckIfIncrementalTs(tsbc.Name)) { throw new NotSupportedException($"Load isn't support for incremental time series '{tsbc.Name}' at document '{tsbc.Id}'"); } } } } BatchOptions options = null; if (Configuration.LoadRequestTimeoutInSec != null) { options = new BatchOptions { RequestTimeout = TimeSpan.FromSeconds(Configuration.LoadRequestTimeoutInSec.Value) }; } using (var batchCommand = new SingleNodeBatchCommand(DocumentConventions.DefaultForServer, context, commands, options)) { var duration = Stopwatch.StartNew(); try { BeforeActualLoad?.Invoke(this); AsyncHelpers.RunSync(() => _requestExecutor.ExecuteAsync(batchCommand, context, token: CancellationToken)); _recentUrl = _requestExecutor.Url; return(commands.Count); } catch (OperationCanceledException e) { if (CancellationToken.IsCancellationRequested == false) { ThrowTimeoutException(commands.Count, duration.Elapsed, e); } throw; } } }
public void QueueLimitSetter_ShouldThrowsException_WhenValueIsNonPositive(int value) { // Arrange var options = new BatchOptions(); // Act + Assert Assert.Throws <ArgumentOutOfRangeException>(() => options.QueueLimit = value); }
private void __bw_DoWork(object sender, DoWorkEventArgs e) { BatchOptions options = e.Argument as BatchOptions; _process.MajorProgressChanged += new EventHandler(process_MajorProgressChanged); _process.MinorProgressChanged += new EventHandler(process_MinorProgressChanged); _process.Process(options); }
public void PeriodSetter_ShouldThrowsException_WhenValueIsNegative(TimeSpan value) { // Arrange var options = new BatchOptions(); // Act + Assert Assert.Throws <ArgumentOutOfRangeException>(() => options.Period = value); }
public void execute(BatchOptions options, string filename) { //todo calc new filename string newFilename = filename; //rename by copy-deleting File.Copy(filename, newFilename); File.Delete(filename); }
public async Task MultiBatchOperationInsertTest() { var inputbatch = new InputMultiBatchOperation { BatchQueries = new BatchOperationQuery[] { new BatchOperationQuery { BatchInputQuery = @"delete from batch_table_test", InputJson = "" }, new BatchOperationQuery { BatchInputQuery = @"insert into batch_table_test (NR,NAM)values(:NR,:NAM)", InputJson = "[{\"NR\": 111, \"NAM\":\"nannaa1\"},{\"NR\":222, \"NAM\":\"nannaa2\"},{\"NR\":333, \"NAM\":\"nannaa3\"}, {\"NR\":444, \"NAM\":\"nannaa4\"}]" }, new BatchOperationQuery { BatchInputQuery = @"insert into batch_table_test (NR,NAM)values(:NR,:NAM)", InputJson = "[{\"NR\": 555, \"NAM\":\"nannaa1\"},{\"NR\":666, \"NAM\":\"nannaa2\"}]" } }, ConnectionString = ConnectionString }; var options = new BatchOptions { ThrowErrorOnFailure = true, IsolationLevel = Oracle_IsolationLevel.Serializable }; MultiBatchOperationOutput output; try { output = await OracleTasks.MultiBatchOperationOracle(inputbatch, options, new CancellationToken()); } catch (Exception ee) { throw ee; } var o = new QueryOutputProperties { ReturnType = QueryReturnType.Json, JsonOutput = new JsonOutputProperties(), OutputToFile = false }; var q2 = new QueryProperties { Query = @"select count(*) as ROWCOUNT from batch_table_test", ConnectionString = ConnectionString }; var options_2 = new QueryOptions(); options.ThrowErrorOnFailure = true; options.IsolationLevel = Oracle_IsolationLevel.Serializable; var result_debug = await OracleTasks.ExecuteQueryOracle(q2, o, options_2, new CancellationToken()); Assert.AreEqual(result_debug.Result, "[\r\n {\r\n \"ROWCOUNT\": 6.0\r\n }\r\n]"); }
public void QueueLimitSetter_ShouldNotThrowsException_WhenValueIsPositive(int?value) { // Arrange var options = new BatchOptions(); // Act options.QueueLimit = value; // Assert Assert.Equal(value, options.QueueLimit); }
public void PeriodSetter_ShouldNotThrowsException_WhenValueIsNotNegative(TimeSpan value) { // Arrange var options = new BatchOptions(); // Act options.Period = value; // Assert Assert.Equal(value, options.Period); }
public void SendAllowsAPartitionHashKeyWithABatch() { var batchOptions = new BatchOptions { PartitionKey = "testKey" }; var batch = new EventDataBatch(new MockTransportBatch(), batchOptions); var transportProducer = new ObservableTransportProducerMock(); var producer = new EventHubProducerClient(new MockConnection(transportProducer)); Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing); }
public void SendAllowsAPartitionHashKeyWithABatch() { var batchOptions = new BatchOptions { PartitionKey = "testKey" }; var batch = new EventDataBatch(new MockTransportBatch(), batchOptions); var transportProducer = new ObservableTransportProducerMock(); var producer = new EventHubProducer(transportProducer, new Uri("amqp://some.endpoint.com/path"), "dummy", new EventHubProducerOptions(), Mock.Of <EventHubRetryPolicy>()); Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing); }
public async Task CreateBatchSetsTheSendOptionsForTheEventBatch() { var batchOptions = new BatchOptions { PartitionKey = "Hi", MaximumSizeInBytes = 9999 }; var transportProducer = new ObservableTransportProducerMock(); var producer = new EventHubProducerClient(new MockConnection(transportProducer)); var eventBatch = await producer.CreateBatchAsync(batchOptions); Assert.That(eventBatch.SendOptions, Is.SameAs(transportProducer.CreateBatchCalledWith), "The batch options should have used for the send options."); ; }
public void CreateBatchForASpecificPartitionDoesNotAllowAPartitionHashKey() { var batchOptions = new BatchOptions { PartitionKey = "testKey" }; var transportProducer = new ObservableTransportProducerMock(); var producer = new EventHubProducer(transportProducer, new Uri("amqp://some.endpoint.com/path"), "dummy", new EventHubProducerOptions { PartitionId = "1" }, Mock.Of <EventHubRetryPolicy>()); Assert.That(async() => await producer.CreateBatchAsync(batchOptions), Throws.InvalidOperationException); }
public async Task CreateBatchSetsTheSendOptionsForTheEventBatch() { var batchOptions = new BatchOptions { PartitionKey = "Hi", MaximumizeInBytes = 9999 }; var transportProducer = new ObservableTransportProducerMock(); var producer = new EventHubProducer(transportProducer, new Uri("amqp://some.endpoint.com/path"), "dummy", new EventHubProducerOptions(), Mock.Of <EventHubRetryPolicy>()); EventDataBatch eventBatch = await producer.CreateBatchAsync(batchOptions); Assert.That(eventBatch.SendOptions, Is.SameAs(transportProducer.CreateBatchCalledWith), "The batch options should have used for the send options."); ; }