public KafkaProducer(Dictionary <string, object> config, string topicName, int partition, EventHandler <IError> dlgtOnError) { this.config = config; this.topicName = topicName; this.partition = partition; this.dlgtOnError = dlgtOnError; var keySerializer = Factory.CreateSerializer <string>(); var valueSerializer = Factory.CreateSerializer <T>(); AssignSerializersIfDefault(ref keySerializer, ref valueSerializer); producer = new Producer <string, T>(config, keySerializer, valueSerializer); threadProcessing = new ThreadProcessing(() => { if (ToBeContinued) { KeyValuePair <string, T> pair; while (GetNextDataPair(out pair)) { producer.ProduceAsync(topicName, pair.Key, pair.Value, partition, blockIfQueueFull: true); } } // Tasks are not waited on synchronously (ContinueWith is not synchronous), // so it's possible they may still in progress here. //producer.Flush(TimeSpan.FromSeconds(10)); }, e => { dlgtOnError(null, new Error((int)ErrorCode.Unknown, e.Message)); LogToScreen.Exception(e); }); }
public StorageWriter(int receiverId, string host, int port, int firstDbNum, int periodInMs, int trimLength, EventHandler <IError> dlgtOnError, string keySuffix, Func <string, T, KeyContext, bool> dlgtFilter = null) { this.keySuffix = keySuffix; // Init DataStore dataStore = DataStore.Create <T>(host, database: firstDbNum, isDurationMeasured: true, port: port); periodicProcessing = new PeriodicProcessing( // PeriodicProc () => { KeyValuePair <string, T> pair; while (cqueData.TryDequeue(out pair)) { try { if (dlgtFilter?.Invoke(pair.Key, pair.Value, GetKeyContext(pair.Key)) != false) { dataStore.Add(pair.Key, pair.Value, trimLength); // Publish DataStore "INSERTED" event dataStore.PublishAsync("ReceiverChannel", $"INSERTED|DATA_RECEIVER|{receiverId}"); } } catch (Exception e) { LogToScreen.Exception(e); } } }, e => dlgtOnError(this, new Error(e)), periodInMs); }
public KafkaConsumer(Dictionary <string, object> config, string topicName, int partition, int offset, MessageProcessingDelegate <T> dlgtMessageProcessing, int consumerTimeoutMs, EventHandler <IError> dlgtOnError) { if (dlgtMessageProcessing == null) { return; } var keyDeserializer = Factory.CreateDeserializer <string>(); var valueDeserializer = Factory.CreateDeserializer <T>(); AssignDeserializersIfDefault(ref keyDeserializer, ref valueDeserializer); consumer = GetConsumer(config, topicName, partition, offset, (sender, e) => dlgtOnError(sender, (Error)e), keyDeserializer, valueDeserializer); consumer.OnMessage += (sender, msg) => { Interlocked.Exchange(ref lastOffset, msg.Offset.Value); Task.Run(() => { try { dlgtMessageProcessing(new Message <T>(msg)); } catch (Exception e) { dlgtOnError(null, new Error((int)ErrorCode.Unknown, e.Message)); LogToScreen.Exception(e); } }); }; threadProcessing = new ThreadProcessing(() => { while (ToBeContinued) { consumer.Poll(consumerTimeoutMs); } }, e => { dlgtOnError(null, new Error((int)ErrorCode.Unknown, e.Message)); LogToScreen.Exception(e); }); }
public StorageReader(Action <int, IList <DSData> > dlgtAction, int processorId, string host, int port, int firstDbNum, int periodInMs, int[] dsRange, int maxLagInMs, EventHandler <IError> dlgtOnError, string keySuffix) { this.keySuffix = keySuffix; // Init DataStore dataStore = DataStore.Create <T>(host, database: firstDbNum, isDurationMeasured: true, port: port); if (periodInMs == Timeout.Infinite) { // Receive DataStore "INSERTED" event dataStore.Subscribe("ReceiverChannel", s => { if (string.IsNullOrEmpty(s)) { return; } var ss = s.Split('|'); int receiverId; if (ss[0] == "INSERTED" && ss[1] == "DATA_RECEIVER" && int.TryParse(ss[2], out receiverId) && processorId == receiverId && periodicProcessing != null) { periodicProcessing.Execute(); } }); } periodicProcessing = new PeriodicProcessing( // PeriodicProc () => { for (int i = dsRange[0]; i <= dsRange[1]; i++) { try { var list = dataStore.Get(GetKeyByDsNum(i)); if (list.Count > 0) { var lstData = new List <DSData>(); foreach (var d in list) { var item = Convert(d); if (item != null) { lstData.Add(item); } } if (lstData.Count > 0) { dlgtAction?.Invoke(i, lstData); // Delay verification if (/*lag*/ DateTime.UtcNow - lstData[0].Timestamp > TimeSpan.FromMilliseconds(maxLagInMs)) { // Lag is too high LogToScreen.Message("?"); } } // Verification of timestamps for LIFO in the list for (int j = 0; j < list.Count - 1; j++) { var d0 = Convert(list[j]); var d1 = Convert(list[j + 1]); if (d0.Timestamp <= d1.Timestamp) { // Wrong chronologic order of data LogToScreen.Message(" ?T "); } } } } catch (Exception e) { LogToScreen.Exception(e); } } }, e => dlgtOnError(this, new Error(e)), periodInMs); }