// Update metadata from latest Kafka records
private void UpdateMetadata()
{
// Attempt to retrieve last known metadata record from Kafka
try
{
using (BrokerRouter router = new BrokerRouter(new KafkaOptions(m_servers)
{
Log = new TimeSeriesLogger(
(status, args) => OnStatusMessage(MessageLevel.Info, string.Format(status, args)),
ex => OnProcessException(MessageLevel.Warning, new InvalidOperationException($"[{MetadataTopic}]: {ex.Message}", ex)))
}))
{
Ticks serializationTime;
OnStatusMessage(MessageLevel.Info, "Reading latest time-series metadata records from Kafka...");
TimeSeriesMetadata metadata = TimeSeriesMetadata.ReadFromKafka(router, MetadataTopic, status => OnStatusMessage(MessageLevel.Info, status), out serializationTime);
if ((object)metadata != null)
{
m_metadata = metadata;
OnStatusMessage(MessageLevel.Info, $"Deserialized {m_metadata.Count:N0} Kafka time-series metadata records, version {m_metadata.Version:N0}, from \"{MetadataTopic}\" serialized at {serializationTime.ToString(MetadataRecord.DateTimeFormat)}");
if (m_lastMetadataVersion != MetadataVersion)
{
m_lastMetadataVersion = MetadataVersion;
m_metadataUpdateCount++;
}
// Cache metadata locally, if configured
m_cacheMetadataLocally?.RunOnceAsync();
}
}
}
catch (Exception ex)
{
if (Enabled)
{
// Treat exception as a warning if metadata already exists
if ((object)m_metadata == null)
{
throw;
}
OnStatusMessage(MessageLevel.Warning, $"Failed to read latest Kafka time-series metadata records from topic \"{MetadataTopic}\": {ex.Message}");
}
}
}
/// <summary>
/// Executes the metadata refresh in a synchronous fashion.
/// </summary>
protected override void ExecuteMetadataRefresh()
{
if (!Initialized || !Enabled || !SerializeMetadata)
{
return;
}
try
{
using (BrokerRouter router = new BrokerRouter(new KafkaOptions(m_servers)
{
Log = new TimeSeriesLogger
(
(status, args) => OnStatusMessage(MessageLevel.Info, string.Format(status, args)),
ex => OnProcessException(MessageLevel.Warning, new InvalidOperationException($"[{MetadataTopic}]: {ex.Message}", ex))
)
}))
{
// Attempt to retrieve last known metadata record from Kafka
if ((object)m_metadata == null)
{
try
{
Ticks serializationTime;
OnStatusMessage(MessageLevel.Info, "Reading latest time-series metadata records from Kafka...");
m_metadata = TimeSeriesMetadata.ReadFromKafka(router, MetadataTopic, status => OnStatusMessage(MessageLevel.Info, status), out serializationTime);
OnStatusMessage(MessageLevel.Info, $"Deserialized {m_metadata.Count:N0} Kafka time-series metadata records, version {m_metadata.Version:N0}, from \"{MetadataTopic}\" serialized at {serializationTime.ToString(MetadataRecord.DateTimeFormat)}");
}
catch (Exception ex)
{
OnStatusMessage(MessageLevel.Warning, $"Failed to read any existing Kafka time-series metadata records from topic \"{MetadataTopic}\": {ex.Message}");
}
}
// Create new meta-data object based on newly loaded configuration
TimeSeriesMetadata metadata = new TimeSeriesMetadata();
try
{
foreach (DataRow row in DataSource.Tables["ActiveMeasurements"].AsEnumerable())
{
MeasurementKey key;
if (MeasurementKey.TryParse(row.Field <string>("ID") ?? MeasurementKey.Undefined.ToString(), out key))
{
metadata.Records.Add(new MetadataRecord
{
ID = key.ID,
Source = key.Source,
UniqueID = row.Field <object>("SignalID").ToString(),
PointTag = row.Field <string>("PointTag"),
Device = row.Field <string>("Device"),
Longitude = row.ConvertField("Longitude", 0.0F),
Latitude = row.ConvertField("Latitude", 0.0F),
Protocol = row.Field <string>("Protocol"),
SignalType = row.Field <string>("SignalType"),
EngineeringUnits = row.Field <string>("EngineeringUnits"),
PhasorType = row.Field <string>("PhasorType"),
Phase = row.Field <string>("Phase"),
Description = row.Field <string>("Description"),
LastUpdate = row.Field <DateTime>("UpdatedOn").ToString(MetadataRecord.DateTimeFormat)
});
}
}
}
catch (Exception ex)
{
OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed to serialize current time-series metadata records: {ex.Message}", ex));
}
if (metadata.Count > 0)
{
// See if metadata has not been created yet or is different from last known Kafka record
if ((object)m_metadata == null || m_metadata.CalculateChecksum() != metadata.CalculateChecksum())
{
// Update local metadata reference
m_metadata = metadata;
// Send updated metadata to Kafka
TimeSeriesMetadata.WriteToKafka(m_metadata, router, MetadataTopic);
// Cache metadata locally, if configured
m_cacheMetadataLocally?.RunOnceAsync();
m_metadataUpdateCount++;
OnStatusMessage(MessageLevel.Info, $"Updated \"{MetadataTopic}\" with {m_metadata.Count:N0} Kafka time-series metadata records...");
}
else
{
OnStatusMessage(MessageLevel.Info, $"Latest \"{MetadataTopic}\" is up to date with current time-series metadata records...");
}
}
else
{
OnStatusMessage(MessageLevel.Warning, "No available local time-series metadata available to serialize...");
}
}
}
catch (Exception ex)
{
OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed to update \"{MetadataTopic}\" with current time-series metadata records: {ex.Message}", ex));
}
}