public static void ReadAndLink()
{
var dependencyUri = new Uri("http://localhost:8180/api/dependencies/"); // The URI where the dependency services are running
const string brokerList = "localhost:9092"; // The host and port where the Kafka broker is running
const string groupName = "dev"; // The group name
const string readTopicName = "sample_in"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the sample_in_announce
const string linkTopicName = "sample_out"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the sample_out_announce
var readAdapter = new KafkaStreamAdapter(brokerList, "readConsumerGroup"); // The steam adapter used to manage streams and topics in the broker. Consumer group must be unique for each stream.
var writeAdapter = new KafkaStreamAdapter(brokerList, "writeConsumerGroup"); // The steam adapter used to manage streams and topics in the broker. Consumer group must be unique for each stream.
var stream = readAdapter.OpenStreamTopic(readTopicName); // Open the topic for streaming.
using (var reader = new Reader(dependencyUri, groupName, stream)) // Create a Reader to read from the stream
{
using (var outputTopic = writeAdapter.OpenOutputTopic(linkTopicName)) // Open the output topic, where you want to link the streamed input
{
using (var writer = new Writer(dependencyUri, AtlasConfiguration, GetDataFormat(), groupName, outputTopic)) // Create a Writer for the output topic and pass
{
const string outputFeedName = ""; // As sample DataFormat uses default feed, we will leave this empty.
using (IStreamPipeline pipeline = reader.ReadAndLinkTSamples(GetParameterIds(), Models.TraceSamples, writer, outputFeedName)) // TelemetryDataHandler parameter can be used to handle the data read from the stream.
{
Thread.Sleep(5000); // NOTE: without this doesn't seem to work
Write(); //Write some data to have something to read while connection is open.
}
}
}
}
}
public static void Write()
{
var dependencyUri = new Uri("http://localhost:8180/api/dependencies/"); // The URI where the dependency services are running
const string brokerList = "localhost:9092"; // The host and port where the Kafka broker is running
const string groupName = "dev"; // The group name
const string topicName = "sample_in"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the sample_in_announce
var streamAdapter = new KafkaStreamAdapter(brokerList, "writeConsumerGroup"); // The steam adapter used to manage streams and topics in the broker. Consumer group must be unique for each stream.
using (var topicOutput = streamAdapter.OpenOutputTopic(topicName)) // Open a KafkaOutputTopic
{
var dataFormat = GetDataFormat(FeedNamFrequencyAndParameterList); // Get the data format for our feed, frequency and parameterId pairs
var feedNames = dataFormat.Feeds.Keys; // Feed names must be the same that we just added above
using (var writer = new Writer(dependencyUri, AtlasConfiguration, dataFormat, groupName, topicOutput)) // Create a Writer object that is used to manage the session and write data
{
writer.OpenSession("sample_" + DateTime.Now); // The session must be opened before Write
for (var i = 0; i < 5; i++) // Write 5 times within the session, but different data.
{
var generatedData = GenerateData(100, (DateTime)writer.SessionStart); // Generate some TelemetryData
foreach (var feedName in feedNames)
{
writer.Write(feedName, generatedData); // Write the TelemetryData to each feed
}
}
writer.CloseSession(); // The session must be closed after writing, otherwise it would be marked as Truncated, which is used as an error flag.
}
}
}
public static void Write()
{
var dependencyUri = new Uri("http://localhost:8180/api/dependencies/"); // The URI where the dependency services are running
const string brokerList = "localhost:9092"; // The host and port where the Kafka broker is running
const string groupName = "dev"; // The group name
const string topicName = "sample_in"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the sample_in_announce
var streamAdapter = new KafkaStreamAdapter(brokerList, "writeConsumerGroup"); // The steam adapter used to manage streams and topics in the broker. Consumer group must be unique for each stream.
using (var topicOutput = streamAdapter.OpenOutputTopic(topicName)) // Open a KafkaOutputTopic
{
using (var writer = new Writer(dependencyUri, AtlasConfiguration, GetDataFormat(), groupName, topicOutput)) // Create a Writer object that is used to manage the session and write data
{
const string feedName = ""; // As sample DataFormat uses default feed, we will leave this empty.
writer.OpenSession("sample_" + DateTime.Now); // The session must be opened before Write
for (var i = 0; i < 5; i++) // Write 5 times within the session, but different data.
{
if (writer.SessionStart == null)
{
continue;
}
var generatedData = GenerateData(100, (DateTime)writer.SessionStart); // Generate some TelemetryData
writer.Write(feedName, generatedData); // Write the TelemetryData to the given feed (default "" in this case)
}
writer.CloseSession(); // The session must be closed after writing, otherwise it would be marked as Truncated, which is used as an error flag.
}
}
}
public static void Read()
{
var dependencyUri = new Uri("http://localhost:8180/api/dependencies/"); // The URI where the dependency services are running
const string brokerList = "localhost:9092"; // The host and port where the Kafka broker is running
const string groupName = "dev"; // The group name
const string topicName = "sample_in"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the sample_in_announce
var streamAdapter = new KafkaStreamAdapter(brokerList, "writeConsumerGroup"); // The steam adapter used to manage streams and topics in the broker. Consumer group must be unique for each stream.
var stream = streamAdapter.OpenStreamTopic(topicName); // Open the topic for streaming.
using (var reader = new Reader(dependencyUri, groupName, stream)) // Create a Reader to read from the stream
{
using (var pipeline = reader.ReadTSamples(GetParameterIds(), Models.TraceSamples)) // TelemetryDataHandler parameter can be used to handle the data read from the stream.
{
Write(); //Write some data to have something to read while connection is open.
}
}
}
