예제 #1
0
        private static Message ReadMessage(BrokerRouter router, string topic, int messageIndex, int maxBytes = 32768)
        {
            Message message = null;

            long offset = router.LatestOffset(topic) - (2 - messageIndex);

            List<Message> messages = router.Fetch(topic, 0, offset, maxBytes)?.Messages;

            if ((messages?.Count ?? 0) > 0)
                message = messages?[0];

            if ((object)message == null)
                throw new InvalidOperationException("No Kafka record to consume");

            return message;
        }
예제 #2
0
        /// <summary>
        /// Writes time-series metadata to specified Kafka <paramref name="topic"/>.
        /// </summary>
        /// <param name="metadata">Source time-series metadata object to write to Kafka.</param>
        /// <param name="router">Kafka router connection.</param>
        /// <param name="topic">Kafka topic.</param>
        public static void WriteToKafka(TimeSeriesMetadata metadata, BrokerRouter router, string topic)
        {
            if ((object)metadata == null)
                throw new ArgumentNullException(nameof(metadata));

            if ((object)router == null)
                throw new ArgumentNullException(nameof(router));

            if (string.IsNullOrWhiteSpace(topic))
                throw new ArgumentNullException(nameof(topic));

            using (MemoryStream stream = new MemoryStream())
            {
                Serialize(metadata, stream, true);

                using (Producer producer = new Producer(router))
                {
                    Message[] messages = new Message[2];
                    byte[] timeKey = BitConverter.GetBytes(DateTime.UtcNow.Ticks);

                    // First message used to serialize metadata size (since metadata messages can be large)
                    messages[SizeMessage] = new Message
                    {
                        Key = timeKey,
                        Value = BitConverter.GetBytes(stream.Length)
                    };

                    // Second message used to serialize metadata value
                    messages[ValueMessage] = new Message
                    {
                        Key = timeKey,
                        Value = stream.ToArray()
                    };

                    // Send meta-data to Kafka
                    producer.SendMessageAsync(topic, messages).Wait();
                }
            }
        }
예제 #3
0
        /// <summary>
        /// Reads latest time-series metadata from specified Kafka <paramref name="topic"/>.
        /// </summary>
        /// <param name="router">Kafka router connection.</param>
        /// <param name="topic">Kafka topic.</param>
        /// <param name="serializationTime">Serialization time.</param>
        /// <param name="statusMessage">Status message function.</param>
        /// <returns>Latest <see cref="TimeSeriesMetadata"/> instance read from Kafka.</returns>
        public static TimeSeriesMetadata ReadFromKafka(BrokerRouter router, string topic, Action<string> statusMessage, out Ticks serializationTime)
        {
            if ((object)router == null)
                throw new ArgumentNullException(nameof(router));

            if (string.IsNullOrWhiteSpace(topic))
                throw new ArgumentNullException(nameof(topic));

            Message sizeMessage = ReadMessage(router, topic, SizeMessage);

            serializationTime = BitConverter.ToInt64(sizeMessage.Key, 0);

            Message valueMessage = ReadMessage(router, topic, ValueMessage, (int)Math.Ceiling(BitConverter.ToInt64(sizeMessage.Value, 0) / 4096.0D) * 4096);

            if (serializationTime != BitConverter.ToInt64(valueMessage.Key, 0))
                statusMessage?.Invoke("WARNING: Timestamp keys for metadata size and value records are mismatched...");

            using (MemoryStream stream = new MemoryStream(valueMessage.Value))
                return Deserialize(stream);
        }
        /// <summary>
        /// Attempts to disconnect from data output stream.
        /// </summary>
        /// <remarks>
        /// Derived classes should attempt disconnect from data output stream here.  Any exceptions thrown
        /// by this implementation will be reported to host via <see cref="AdapterBase.ProcessException"/> event.
        /// </remarks>
        protected override void AttemptDisconnection()
        {
            if ((object)m_producer != null)
            {
                m_producer.Dispose();
                m_producer = null;
            }

            if ((object)m_router != null)
            {
                m_router.Dispose();
                m_router = null;
            }
        }
        /// <summary>
        /// Executes the metadata refresh in a synchronous fashion.
        /// </summary>
        protected override void ExecuteMetadataRefresh()
        {
            if (!Initialized || !Enabled || !SerializeMetadata)
                return;

            try
            {
                using (BrokerRouter router = new BrokerRouter(new KafkaOptions(m_servers)
                {
                    Log = new TimeSeriesLogger
                    (
                        (status, args) => OnStatusMessage(MessageLevel.Info, string.Format(status, args)),
                        ex => OnProcessException(MessageLevel.Warning, new InvalidOperationException($"[{MetadataTopic}]: {ex.Message}", ex))
                    )
                }))
                {
                    // Attempt to retrieve last known metadata record from Kafka
                    if ((object)m_metadata == null)
                    {
                        try
                        {
                            Ticks serializationTime;

                            OnStatusMessage(MessageLevel.Info, "Reading latest time-series metadata records from Kafka...");

                            m_metadata = TimeSeriesMetadata.ReadFromKafka(router, MetadataTopic, status => OnStatusMessage(MessageLevel.Info, status), out serializationTime);

                            OnStatusMessage(MessageLevel.Info, $"Deserialized {m_metadata.Count:N0} Kafka time-series metadata records, version {m_metadata.Version:N0}, from \"{MetadataTopic}\" serialized at {serializationTime.ToString(MetadataRecord.DateTimeFormat)}");
                        }
                        catch (Exception ex)
                        {
                            OnStatusMessage(MessageLevel.Warning, $"Failed to read any existing Kafka time-series metadata records from topic \"{MetadataTopic}\": {ex.Message}");
                        }
                    }

                    // Create new meta-data object based on newly loaded configuration
                    TimeSeriesMetadata metadata = new TimeSeriesMetadata();

                    try
                    {
                        foreach (DataRow row in DataSource.Tables["ActiveMeasurements"].AsEnumerable())
                        {
                            MeasurementKey key;

                            if (MeasurementKey.TryParse(row.Field<string>("ID") ?? MeasurementKey.Undefined.ToString(), out key))
                            {
                                metadata.Records.Add(new MetadataRecord
                                {
                                    ID = key.ID,
                                    Source = key.Source,
                                    UniqueID = row.Field<object>("SignalID").ToString(),
                                    PointTag = row.Field<string>("PointTag"),
                                    Device = row.Field<string>("Device"),
                                    Longitude = row.ConvertField("Longitude", 0.0F),
                                    Latitude = row.ConvertField("Latitude", 0.0F),
                                    Protocol = row.Field<string>("Protocol"),
                                    SignalType = row.Field<string>("SignalType"),
                                    EngineeringUnits = row.Field<string>("EngineeringUnits"),
                                    PhasorType = row.Field<string>("PhasorType"),
                                    Phase = row.Field<string>("Phase"),
                                    Description = row.Field<string>("Description"),
                                    LastUpdate = row.Field<DateTime>("UpdatedOn").ToString(MetadataRecord.DateTimeFormat)
                                });
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed to serialize current time-series metadata records: {ex.Message}", ex));
                    }

                    if (metadata.Count > 0)
                    {
                        // See if metadata has not been created yet or is different from last known Kafka record
                        if ((object)m_metadata == null || m_metadata.CalculateChecksum() != metadata.CalculateChecksum())
                        {
                            // Update local metadata reference
                            m_metadata = metadata;

                            // Send updated metadata to Kafka
                            TimeSeriesMetadata.WriteToKafka(m_metadata, router, MetadataTopic);

                            // Cache metadata locally, if configured
                            m_cacheMetadataLocally?.RunOnceAsync();

                            m_metadataUpdateCount++;

                            OnStatusMessage(MessageLevel.Info, $"Updated \"{MetadataTopic}\" with {m_metadata.Count:N0} Kafka time-series metadata records...");
                        }
                        else
                        {
                            OnStatusMessage(MessageLevel.Info, $"Latest \"{MetadataTopic}\" is up to date with current time-series metadata records...");
                        }
                    }
                    else
                    {
                        OnStatusMessage(MessageLevel.Warning, "No available local time-series metadata available to serialize...");
                    }
                }
            }
            catch (Exception ex)
            {
                OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed to update \"{MetadataTopic}\" with current time-series metadata records: {ex.Message}", ex));
            }
        }
 /// <summary>
 /// Attempts to connect to data output stream.
 /// </summary>
 /// <remarks>
 /// Derived classes should attempt connection to data output stream here.  Any exceptions thrown
 /// by this implementation will result in restart of the connection cycle.
 /// </remarks>
 protected override void AttemptConnection()
 {
     m_router = new BrokerRouter(new KafkaOptions(m_servers)
     {
         Log = new TimeSeriesLogger
         (
             (status, args) => OnStatusMessage(MessageLevel.Info, string.Format(status, args)), 
             ex => OnProcessException(MessageLevel.Warning, ex)
         )
     });
     m_producer = new Producer(m_router);
     MetadataRefreshOperation.RunOnceAsync();
 }
예제 #7
0
        static void Main(string[] args)
        {

            var options = new KafkaOptions(new Uri("http://kafka1:9092"), new Uri("http://kafka2:9092"))
            {
                Log = new ConsoleLog()
            };
            var router = new BrokerRouter(options);
            var client = new Producer(router);
            var timing = new RollingQueue<double>(50);
            var rate = new RollingQueue<double>(50);
            var second = DateTime.Now.Second;
            var count = 0;
            

            Task.Run(() =>
            {
                var consumer = new Consumer(new ConsumerOptions("latencies", router));
                var position = consumer.GetTopicOffsetAsync("latencies");
                position.Wait();
                consumer.SetOffsetPosition(
                    position.Result
                        .Select(p => new OffsetPosition(p.PartitionId, p.Offsets.First()))
                        .ToArray()
                    );

                foreach (var data in consumer.Consume())
                {
                    count++;
                    var rtt = (DateTime.Now - new DateTime(
                        long.Parse(Encoding.UTF8.GetString(data.Value))
                        )).TotalMilliseconds;

                    if (rtt < 1000)
                        timing.Enqueue(rtt);

                    if (second != DateTime.Now.Second)
                    {
                        second = DateTime.Now.Second;
                        rate.Enqueue(count);
                        count = 0;
                        Console.WriteLine("Rate: {0} pps.\t{1} ", 
                            rate.Average().ToString("N2") , (rtt < 1000) 
                            ? "RTT: " + timing.Average().ToString("N2") + " ms."
                            : string.Empty
                        );
                    }
                    
                }
            });



            while (true)
            {
                client.SendMessageAsync("latencies", new[] {
                    new Message(DateTime.Now.Ticks.ToString()),
                    new Message(DateTime.Now.Ticks.ToString()),
                    new Message(DateTime.Now.Ticks.ToString()),
                    new Message(DateTime.Now.Ticks.ToString()),
                    new Message(DateTime.Now.Ticks.ToString()),
                    new Message(DateTime.Now.Ticks.ToString()),
                    new Message(DateTime.Now.Ticks.ToString()),
                    new Message(DateTime.Now.Ticks.ToString()),
                    new Message(DateTime.Now.Ticks.ToString()),
                    new Message(DateTime.Now.Ticks.ToString())
                }, 1);
                Thread.Sleep(1);
            }

            client.Dispose();
            router.Dispose();
        }
        // Update metadata from latest Kafka records
        private void UpdateMetadata()
        {
            // Attempt to retrieve last known metadata record from Kafka
            try
            {
                using (BrokerRouter router = new BrokerRouter(new KafkaOptions(m_servers)
                {
                    Log = new TimeSeriesLogger(
                        (status, args) => OnStatusMessage(MessageLevel.Info, string.Format(status, args)), 
                        ex => OnProcessException(MessageLevel.Warning, new InvalidOperationException($"[{MetadataTopic}]: {ex.Message}", ex)))
                }))
                {
                    Ticks serializationTime;

                    OnStatusMessage(MessageLevel.Info, "Reading latest time-series metadata records from Kafka...");

                    TimeSeriesMetadata metadata = TimeSeriesMetadata.ReadFromKafka(router, MetadataTopic, status => OnStatusMessage(MessageLevel.Info, status), out serializationTime);

                    if ((object)metadata != null)
                    {
                        m_metadata = metadata;

                        OnStatusMessage(MessageLevel.Info, $"Deserialized {m_metadata.Count:N0} Kafka time-series metadata records, version {m_metadata.Version:N0}, from \"{MetadataTopic}\" serialized at {serializationTime.ToString(MetadataRecord.DateTimeFormat)}");

                        if (m_lastMetadataVersion != MetadataVersion)
                        {
                            m_lastMetadataVersion = MetadataVersion;
                            m_metadataUpdateCount++;
                        }

                        // Cache metadata locally, if configured
                        m_cacheMetadataLocally?.RunOnceAsync();
                    }
                }
            }
            catch (Exception ex)
            {
                if (Enabled)
                {
                    // Treat exception as a warning if metadata already exists
                    if ((object)m_metadata == null)
                        throw;

                    OnStatusMessage(MessageLevel.Warning, $"Failed to read latest Kafka time-series metadata records from topic \"{MetadataTopic}\": {ex.Message}");
                }
            }
        }
        /// <summary>
        /// Attempts to disconnect from data output stream.
        /// </summary>
        /// <remarks>
        /// Derived classes should attempt disconnect from data output stream here.  Any exceptions thrown
        /// by this implementation will be reported to host via <see cref="AdapterBase.ProcessException"/> event.
        /// </remarks>
        protected override void AttemptDisconnection()
        {
            if ((object)m_processingThreads != null)
            {
                foreach (Thread processingThread in m_processingThreads)
                    processingThread.Abort();

                m_processingThreads = null;
            }

            if ((object)m_router != null)
            {
                m_router.Dispose();
                m_router = null;
            }

            lock (m_consumers)
                m_consumers.Clear();
        }
        /// <summary>
        /// Attempts to connect to data output stream.
        /// </summary>
        /// <remarks>
        /// Derived classes should attempt connection to data output stream here.  Any exceptions thrown
        /// by this implementation will result in restart of the connection cycle.
        /// </remarks>
        protected override void AttemptConnection()
        {
            m_router = new BrokerRouter(new KafkaOptions(m_servers)
            {
                Log = new TimeSeriesLogger
                (
                    (status, args) => OnStatusMessage(MessageLevel.Info, string.Format(status, args)), 
                    ex => OnProcessException(MessageLevel.Warning, ex)
                )
            });

            m_processingThreads = new Thread[Partitions];

            for (int partition = 0; partition < m_processingThreads.Length; partition++)
            {
                m_processingThreads[partition] = new Thread(ProcessPartitionMessages) { IsBackground = true };
                m_processingThreads[partition].Start(partition);
            }

            // Kick off process to update metadata
            m_updateMetadata.RunOnceAsync();
        }