public override ConsumeResult Consume(LeskContext context) { var result = new ConsumeResult(); var match = RegexLookup[Pattern].Regex.Match(context.Input, context.Position); if (match.Success && match.Index == context.Position && match.Length > 0) { result.Success = true; result.Consumed = match.Value; } return result; }
public ValueTask EnqueueAsync( ConsumeResult <byte[], byte[]> message, CancellationToken stopCancellationToken = default) { return(this.messagesBuffer.Writer.WriteAsync(message, stopCancellationToken)); }
private void ProcessMessage(ConsumeResult <Key, Event> consumeResult) { _logger.LogInformation($"Kafka Consumer service received the message '{consumeResult.Value.message.Body}' from '{consumeResult.TopicPartitionOffset}'."); }
private Context CreateNewExecutionContext(ConsumeResult <string, TPayload> consumeResult) => new Context { { "consume-result", consumeResult } };
public static void Consumer_Exiting(string bootstrapServers, string singlePartitionTopic, string partitionedTopic) { LogToFile("start Consumer_Consume"); int N = 2; var firstProduced = Util.ProduceMessages(bootstrapServers, singlePartitionTopic, 100, N); var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, SessionTimeoutMs = 6000 }; for (int i = 0; i < 4; ++i) { using (var consumer = new Consumer <Null, string>(consumerConfig)) { consumer.OnPartitionsAssigned += (_, partitions) => consumer.Assign(partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset))); consumer.OnPartitionsRevoked += (_, partitions) => consumer.Unassign(); consumer.Subscribe(singlePartitionTopic); int tryCount = 10; while (tryCount-- > 0) { ConsumeResult <Null, string> record = consumer.Consume(TimeSpan.FromMilliseconds(1000)); if (record != null) { break; } } Assert.True(tryCount > 0); // there should be no ill effect doing any of this before disposing a consumer. switch (i) { case 0: consumer.Unsubscribe(); break; case 1: consumer.Commit(); break; case 3: consumer.Close(); break; case 4: break; } } } Assert.Equal(0, Library.HandleCount); LogToFile("end Consumer_Consume"); }
private static void ConsumeAsUtf8Json(ConsumeResult <Ignore, byte[]> consumeResult) => Consume(consumeResult, JsonSerializer.Deserialize <Block>, JsonSerializer.Deserialize <Transaction>, JsonSerializer.Deserialize <TransactionReceipt>);
internal void SetRecordMetaData(ConsumeResult <byte[], byte[]> result) { RecordContext = new RecordContext(result); }
public IEnumerable <object> GetKafkaObjets() { IConsumer <byte[], string> consumer = null; try { try { //creating the consumer consumer = new ConsumerBuilder <byte[], string>(_consumerConfig).SetKeyDeserializer(Deserializers.ByteArray).Build(); //subscribing to the topic from the jobData consumer.Subscribe(this._kafkaCrawlJobData.KafkaTopic); } catch (Exception ex) { this.log.Error(() => $"Kafka Crawler - Could not create Consumer. Exception: {ex.Message}"); if (consumer != null) { consumer.Dispose(); } yield break; } //track the time spent in the loop var stopWatch = new Stopwatch(); stopWatch.Start(); var attempt = 1; while (!_cts.IsCancellationRequested) { ConsumeResult <byte[], string> cr = null; try { cr = consumer.Consume(new TimeSpan(0)); attempt = 1; } catch (Exception ex) { this.log.Error(() => $"Kafka Crawler - Could not consume. Exception: {ex.Message}. Attempt {attempt}."); attempt++; if (attempt > 3) { this.log.Error(() => $"Kafka Crawler - Could not consume. Exception: {ex.Message}. Ending loop."); _cts.Cancel(); } else { Thread.Sleep(1000); } continue; } if (cr?.Message != null) { stopWatch.Restart(); //if we found a message, we restart the timer to 0 var resource = this.GetTypedObject(cr); if (resource == null) { continue; } yield return(resource); } else if (stopWatch.Elapsed.TotalMinutes >= this._kafkaCrawlJobData.KafkaDummyClueGenerationInterval) //in order to prevent the job from being shut down, we create a dummy clue and then we restart the timer { stopWatch.Restart(); this.log.Info(() => $"Kafka Crawler - Creating dummy object after receiving 0 messages for 5 minutes."); yield return(new Contact { AccountId = $"Dummy AccountId", AccountIdName = $"Dummy AccountIdName", AccountIdYomiName = $"Dummy AccountIdName", AccountRoleCode = $"Dummy AccountIdName", AccountRoleCodeName = $"Dummy AccountIdName", ContactId = $"Dummy ContactId", Description = $"Dummy Description", FullName = $"Dummy FullName", NickName = $"Dummy NickName", }); } } consumer.Dispose(); } finally { if (consumer != null) { consumer.Dispose(); } } }
async Task <CancellationToken> IKafkaProcessor <Rootobject> .ProcessMessage(Rootobject t, ConsumeResult <Ignore, string> MessageDetails, CancellationToken Token) { CancellationTokenSource source = new CancellationTokenSource(); Token = source.Token; if (t.FileName.Contains("cancel")) { source.Cancel(); } return(Token); }
/// <summary> /// TransactionalMessage /// </summary> public TransactionalMessage(ConsumeResult <K, V> record, GroupTopicPartitionOffset partitionOffset) { Record = record; PartitionOffset = partitionOffset; }
private long GetFileSize(ConsumeResult <string, byte[]> receivedMessage) { return(BitConverter.ToInt64(receivedMessage.Value, 0)); }
protected override Task OnError(Exception exception, ConsumeResult <string, string> result) { OnErrorEvent.Invoke(exception, result); return(Task.CompletedTask); }
protected override Task OnConsume(ConsumeResult <string, string> result) { OnConsumeEvent.Invoke(result); return(Task.CompletedTask); }
/// <summary> /// Method for consuming a new message on the system topic time /// </summary> /// <param name="cancelToken">The cancellation token to give to the consumer</param> private void ConsumeTimeControlMessage(CancellationToken cancelToken) { try { while (true) { try { ConsumeResult <EDXLDistribution, TimeControl> res = _timeControlConsumer.Consume(cancelToken); // Update the values of the time info switch (res.Value.command) { case TimeCommand.Init: _currentTime.TimeState = TimeState.Initialization; break; case TimeCommand.Start: _currentTime.TimeState = TimeState.Started; break; case TimeCommand.Update: break; case TimeCommand.Pause: _currentTime.TimeState = TimeState.Paused; break; case TimeCommand.Stop: _currentTime.TimeState = TimeState.Stopped; break; case TimeCommand.Reset: _currentTime.TimeState = TimeState.Reset; break; } if (res.Value.simulationTime.HasValue) { TimeSpan trialTime = TimeSpan.FromMilliseconds(res.Value.simulationTime.Value); _currentTime.TrialTime = UNIXEpoch.Add(trialTime); } if (res.Value.simulationSpeed.HasValue) { _currentTime.TrialTimeSpeed = res.Value.simulationSpeed.Value; } // If the callback handler was provided, notify the application of the timing control change if (_timingControlHandler != null) { _timingControlHandler.Invoke(); } } catch (ConsumeException e) { throw new CommunicationException($"consume error, {e.Error.Reason}"); } } } catch (OperationCanceledException) { _timeControlConsumer.Close(); } }
/// <inheritdoc /> public void ConsumeException(Exception exception, IMessageContext messageContext, ConsumeResult consumeResult) { var contextLogger = seriLogger .ForContext("consumeResult", consumeResult) .ForContext("exchange", messageContext.Exchange) .ForContext("queue", messageContext.Queue) .ForContext("routingKey", messageContext.RoutingKey); if (messageContext is IControllerMessageContext controllerMessageContext) { contextLogger = contextLogger .ForContext("controller", controllerMessageContext.Binding.Controller.FullName) .ForContext("method", controllerMessageContext.Binding.Method.Name); } contextLogger.Error(exception, "Tapeti: exception in message handler"); }
public void setWriteResult(ConsumeResult <string, GenericRecord> request, string outcome) { key = request.Message.Key; trySetID(request.Message.Value, request.Offset.Value); result = outcome; }
public void MessageReceivedExceptionOcurred(ConsumeResult <TKey, TValue> result, Exception e) { Retry(result); this.consumerClient.MessageReceivedExceptionOcurred(result, e); }
static void KafkaLongRunningProcessTest(EmployeeDbContext context, string arg) { var topic = "SampleEvent"; var conf = new ConsumerConfig { GroupId = "sample-event-consumer-group-2", BootstrapServers = "localhost:9092", AutoOffsetReset = AutoOffsetReset.Earliest, EnableAutoCommit = true, EnableAutoOffsetStore = false, MaxPollIntervalMs = 86400000 //MaxPollIntervalMs = 12000, //AutoCommitIntervalMs = 3000 }; using (var consumer = new ConsumerBuilder <Ignore, string>(conf).Build()) { consumer.Subscribe(topic); CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); }; try { ConsumeResult <Ignore, string> result = null; while (true) { try { result = consumer.Consume(cts.Token); Employee employee = Get(context); Console.WriteLine( $"Consumed message '{ result.Message.Value }' at: { DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss,fff") }'." ); // Test 100 seconds, 1 minute 40 seconds - works // Test 200 seconds, 3 minutes 20 seconds - works // Test 400 seconds, 6 minutes 40 seconds - Hit error: Application maximum poll interval (300000ms) // exceeded by 98ms (adjust max.poll.interval.ms for long-running message processing): leaving group // Resolve by setting MaxPollIntervalMs = 86400000 // Test 800 seconds, 13 minutes 20 seconds - works // Test 3000 seconds, 50 minutes - works Thread.Sleep(TimeSpan.FromMilliseconds(30000)); Update(context, employee, arg); //consumer.Commit(result); // sync commit will failed if consumer leave group consumer.StoreOffset(result); // async commit with background thread Console.WriteLine( $"Commit: { DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss,fff") }" ); } catch (ConsumeException ex) { Console.WriteLine($"Error occured: { ex.Error.Reason }"); } catch (TopicPartitionException ex) { Console.WriteLine($"Commit error: { ex.Error.Reason }"); } catch (KafkaException ex) { Console.WriteLine($"Commit error: { ex.Error.Reason }"); } catch (DbUpdateConcurrencyException ex) { consumer.StoreOffset(result); ex.Entries.Single().Reload(); Console.WriteLine($"DbUpdateConcurrencyException error: { ex.Message }"); } catch (Exception ex) { Console.WriteLine($"General error: { ex.Message }"); } } } catch (OperationCanceledException) { // Ensure the consumer leaves the group cleanly and final offsets are committed. consumer.Close(); } finally { consumer.Close(); } } }
public KafkaConsumeContext(ReceiveContext receiveContext, ConsumeResult <TKey, TValue> result) : base(receiveContext) { Message = result.Message.Value; _adapter = new KafkaHeaderAdapter <TKey, TValue>(result, receiveContext); }
public KafkaMessageContext(IServiceProvider serviceProvider, ConsumeResult <TKey, TValue> consumeResult) : base(serviceProvider) { this.SetKafkaMessage(consumeResult); }
public abstract long onInvalidTimestamp(ConsumeResult <object, object> record, long recordTimestamp, long partitionTime);
private IMessage ToMessage(ConsumeResult <string, string> record) { return(JsonMapper.FromJson <Message>(record.Value)); }
private static void StartConsumingAvro <T>(string topic) where T : ISpecificRecord { bool consuming = true; ConsumerBuilder <Ignore, T> consumerBuilder = new ConsumerBuilder <Ignore, T>(Configuration.ConsumerConfig); consumerBuilder.SetErrorHandler((s, e) => { consuming = !e.IsFatal; Log(e.ToString()); }); CachedSchemaRegistryClient schemaRegistry = new CachedSchemaRegistryClient(new[] { new KeyValuePair <string, string>(SchemaRegistryConfig.PropertyNames.SchemaRegistryUrl, Configuration.SchemaRegistryUrl) }); var deserializer = new AvroDeserializer <T>(schemaRegistry).AsSyncOverAsync(); consumerBuilder.SetValueDeserializer(deserializer); IConsumer <Ignore, T> consumer = consumerBuilder.Build(); using (consumer) { ConsumerConfig consumerConfig = Configuration.ConsumerConfig; Log($"Consumer for group: '{consumerConfig.GroupId}' was created. Data type: '{Configuration.Type}'."); consumer.Subscribe(new[] { topic }); Log($"Subscribed to topic: '{topic}'."); while (consuming) { try { ConsumeResult <Ignore, T> consumeResult = consumer.Consume(); Type type = typeof(T); if (type == typeof(Avro.Models.Block)) { ConsumeResult <Ignore, Avro.Models.Block> result = consumeResult as ConsumeResult <Ignore, Avro.Models.Block>; Avro.Models.Block block = result.Value; Log($"Block: {block.blockNumber} {block.blockHash}"); } else if (type == typeof(Avro.Models.FullTransaction)) { ConsumeResult <Ignore, FullTransaction> result = consumeResult as ConsumeResult <Ignore, Avro.Models.FullTransaction>; FullTransaction transaction = result.Value; Log($"Transaction for block: {transaction.blockNumber} {transaction.receipt.blockHash}"); } else { Log($"Unknown data type: {type.Name}"); continue; } Log($"Consumed value at '{consumeResult.TopicPartitionOffset}'."); } catch (ConsumeException exception) { Log($"Consumer error occured: {exception.Error.Reason}"); } catch (Exception exception) { Log(exception.Message); } } consumer.Close(); } }
public KafkaHeaderAdapter(ConsumeResult <TKey, TValue> result, ReceiveContext receiveContext) { _result = result; _receiveContext = receiveContext; }
private async void Consumer_Received(object sender, ConsumeResult <string, byte[]> eventArgs) { var kafkaMessage = eventArgs.Message; var messageTypeFullName = string.Empty; foreach (var header in kafkaMessage.Headers) { if (header.Key != "MessageType") { continue; } messageTypeFullName = Encoding.UTF8.GetString(header.GetValueBytes()); break; } var typeResolver = _serviceProvider.GetService <ITypeResolver>(); if (!typeResolver.TryResolveType(messageTypeFullName, out var messageType)) { _logger.LogError($"The Consume Group: {_consumer.Group} received kafka message from topic: {eventArgs.Topic}, but cannot resolve type: {messageTypeFullName}."); return; } ; var payload = _serializer.Deserialize(eventArgs.Message.Value, messageType); if (!(payload is IMessage message)) { _logger.LogError($"The Consume Group: {_consumer.Group} deserialized kafka message from topic: {eventArgs.Topic}, but incorrect message type, expected: {messageTypeFullName}, actual: {payload.GetType().FullName}."); return; } ; var handlerProxyTypeDefinition = typeof(IMessageHandlerProxy <>); var asyncHandlerProxyTypeDefinition = typeof(IAsyncMessageHandlerProxy <>); using (var scope = _serviceProvider.CreateScope()) { var hasHandler = false; var scopeServiceProvider = scope.ServiceProvider; var handlerProxies = scopeServiceProvider.GetServices(handlerProxyTypeDefinition.MakeGenericType(messageType)); if (handlerProxies.IsNotEmpty()) { hasHandler = true; MessageHandlerUtils.DynamicInvokeHandle(handlerProxies, message, _logger); } var asyncHandlerProxies = scopeServiceProvider.GetServices(asyncHandlerProxyTypeDefinition.MakeGenericType(messageType)); if (asyncHandlerProxies.IsNotEmpty()) { hasHandler = true; await MessageHandlerUtils.DynamicInvokeAsyncHandle(asyncHandlerProxies, message, _logger); } if (!hasHandler) { _logger.LogWarning($"No message handler found for {messageTypeFullName}, the consume group: {_consumer.Group}, topic: {eventArgs.Topic}."); } } }
internal KafkaReceiverMessage(IConsumer <Ignore, string> consumer, ConsumeResult <Ignore, string> result) : base(() => result.Value) { Consumer = consumer; Result = result; }
/// <summary> /// Begins consumption from Kafka /// </summary> internal void Start() { // UDP IPEndPoint sockEP = new IPEndPoint(IPAddress.Loopback, WEBSOCK_PORT); UdpClient socket = new UdpClient(sockEP); // Consumer object using var consumer = this.KafkaConsumer; // Subscribe consumer to VATSIM kafka feed consumer.Subscribe("datafeed"); // Ensure safe termination CancellationTokenSource tokenSource = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => // Wait for Ctrl+C { // Cancel kafka consumption safely e.Cancel = true; tokenSource.Cancel(); }; // Read from kafka try { // Consume until cancel triggered while (true) { try { // Consume Kafka ConsumeResult <Ignore, string> result = consumer.Consume(tokenSource.Token); // Get the raw message JObject messageRaw = JObject.Parse(result.Message.Value); // Check if the message type is correct if ((string)messageRaw["message_type"] == "update_position") { // Cast coordinates to doubles double lat = (double)messageRaw["data"]["latitude"]; double lon = (double)messageRaw["data"]["longitude"]; double altitudeSI = (int)messageRaw["data"]["altitude"] / 3.2808; // Define new GeoJson point Point geoPoint = new Point(new Position(lat, lon, altitudeSI)); // Built PilotPosition object PilotPosition position = new PilotPosition( (string)messageRaw["data"]["callsign"], altitudeSI, // metres geoPoint); // *Sigh* DefaultContractResolver camelCase = new DefaultContractResolver { NamingStrategy = new SnakeCaseNamingStrategy() }; // Serialise the pilot position string jsonPosition = JsonConvert.SerializeObject(position, new JsonSerializerSettings { ContractResolver = camelCase }); // Write to console Console.WriteLine(jsonPosition + "\n"); // Write datagram and send byte[] bytesToSend = Encoding.ASCII.GetBytes(jsonPosition); socket.Send(bytesToSend, bytesToSend.Length, sockEP); } else // If not a pilot position update { // We don't want it continue; } } catch (ConsumeException ex) // If something happens { // Catch the oops Console.WriteLine($"Exception: {ex.Error.Reason}"); } } } catch (OperationCanceledException) // Catch the cancel event { // Leave group and close cleanly consumer.Close(); } }
public void Consumer_Pause_Resume(string bootstrapServers) { LogToFile("start Consumer_Pause_Resume"); var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, AutoOffsetReset = AutoOffsetReset.Latest }; var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; IEnumerable <TopicPartition> assignment = null; using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var producer = new ProducerBuilder <byte[], byte[]>(producerConfig).Build()) using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig) .SetPartitionsAssignedHandler((c, partitions) => { assignment = partitions; }) .Build()) { consumer.Subscribe(topic.Name); while (assignment == null) { consumer.Consume(TimeSpan.FromSeconds(2)); } ConsumeResult <byte[], byte[]> record = consumer.Consume(TimeSpan.FromSeconds(2)); Assert.Null(record); producer.ProduceAsync(topic.Name, new Message <byte[], byte[]> { Value = Serializers.Utf8.Serialize("test value", SerializationContext.Empty) }).Wait(); record = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.NotNull(record?.Message); Assert.Equal(0, record?.Offset); consumer.Pause(assignment); producer.ProduceAsync(topic.Name, new Message <byte[], byte[]> { Value = Serializers.Utf8.Serialize("test value 2", SerializationContext.Empty) }).Wait(); record = consumer.Consume(TimeSpan.FromSeconds(2)); Assert.Null(record); consumer.Resume(assignment); record = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.NotNull(record?.Message); Assert.Equal(1, record?.Offset); // check that these don't throw. consumer.Pause(new List <TopicPartition>()); consumer.Resume(new List <TopicPartition>()); consumer.Close(); } Assert.Equal(0, Library.HandleCount); LogToFile("end Consumer_Pause_Resume"); }
public static int Main(string[] args) { var show_help = false; var use_precompiled = true; var options = new OptionSet { { "p|no-precomp", "do not use precompiled libraries", v => use_precompiled = v == null }, { "h|help", "show this message and exit", v => show_help = v != null } }; List <string> files; try { files = options.Parse(args); } catch (OptionException e) { Console.Error.Write(AppDomain.CurrentDomain.FriendlyName + ": "); Console.Error.WriteLine(e.Message); Console.Error.WriteLine("Try “" + AppDomain.CurrentDomain.FriendlyName + " --help” for more information."); return(1); } if (show_help) { Console.WriteLine("Usage: " + AppDomain.CurrentDomain.FriendlyName + " input.flbgst"); Console.WriteLine("Run Flabbergast interactively."); Console.WriteLine(); Console.WriteLine("Options:"); options.WriteOptionDescriptions(Console.Out); return(1); } if (files.Count > 1) { Console.Error.WriteLine("No more than one Flabbergast script may be given."); return(1); } Frame original = null; var assembly_builder = AppDomain.CurrentDomain.DefineDynamicAssembly(new AssemblyName("Repl"), AssemblyBuilderAccess.Run); var module_builder = assembly_builder.DefineDynamicModule("ReplModule"); var unit = new CompilationUnit(module_builder, false); var collector = new ConsoleCollector(); var task_master = new ConsoleTaskMaster(); task_master.AddUriHandler(BuiltInLibraries.INSTANCE); if (use_precompiled) { task_master.AddUriHandler(new LoadPrecompiledLibraries()); } task_master.AddUriHandler(new DynamicallyCompiledLibraries(collector)); if (files.Count == 1) { var parser = Parser.Open(files[0]); var root_type = parser.ParseFile(collector, unit, "REPLRoot"); if (root_type != null) { var computation = (Computation)Activator.CreateInstance(root_type, task_master); computation.Notify(r => original = r as Frame); task_master.Slot(computation); task_master.Run(); task_master.ReportCircularEvaluation(); } } if (original == null) { original = new Frame(task_master, task_master.NextId(), new SourceReference("<repl>", "<native>", 0, 0, 0, 0, null), null, null); } var id = 0; Frame current = original; bool run = true; ConsumeResult update_current = (x) => current = (x as Frame) ?? current; var line_editor = new LineEditor("flabbergast"); var completables = new Completables(); line_editor.AutoCompleteEvent = completables.Handler; string s; while (run && (s = line_editor.Edit(id + "‽ ", "")) != null) { var parser = new Parser("line" + id, s); var run_type = parser.ParseRepl(collector, unit, "REPL" + id++); if (run_type != null) { object result = null; var computation = (Computation)Activator.CreateInstance(run_type, new object[] { task_master, original, current, update_current, (ConsumeResult)(output => result = output), (ConsumeResult)Console.WriteLine }); computation.Notify(r => run = (r as bool?) ?? true); task_master.Slot(computation); task_master.Run(); if (result != null) { HandleResult(result); } task_master.ReportCircularEvaluation(); } } line_editor.SaveHistory(); return(0); }
private void Resend(ConsumeResult <TKey, TValue> cr, string topic) { retrySender.SendAsync(cr, topic).Wait(); }
/** * Attach a callback when the computation is complete. If already complete, * the callback is immediately invoked. */ public void Notify(ConsumeResult new_consumer) { Notify(new_consumer, true); }
/** * Access a value if available, or be notified upon completion. * Returns: true if the value was available, false if the caller should wait to be reinvoked. */ internal bool GetOrSubscribe(string name, ConsumeResult consumer) { // If this frame is being looked at, then all its pending attributes should // be slotted. Slot(); object value; if (attributes.TryGetValue(name, out value)) { if (value is Computation) { ((Computation) value).Notify(consumer); } else { consumer(value); } return true; } return false; }