public void WriteEvents()
{
const string brokerList = "localhost:9092"; // The host and port where the Kafka broker is running
const string groupName = "dev"; // The dependency group name
const string topicName = "events_sample"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the data_in_announce
var dependencyServiceUri = new Uri("http://localhost:8180/api/dependencies/"); // The URI where the dependency services are running
var client = new KafkaStreamClient(brokerList); // Create a new KafkaStreamClient for connecting to Kafka broker
var dataFormatClient = new DataFormatClient(new HttpDependencyClient(dependencyServiceUri, groupName)); // Create a new DataFormatClient
var httpDependencyClient = new HttpDependencyClient(dependencyServiceUri, groupName); // DependencyClient stores the Data format, Atlas Configuration
var atlasConfigurationId = new AtlasConfigurationClient(httpDependencyClient).PutAndIdentifyAtlasConfiguration(AtlasConfiguration); // Uniq ID created for the AtlasConfiguration
var dataFormat = DataFormat.DefineFeed().BuildFormat(); // Create a dataformat based on the parameters, using the parameter id
var dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(dataFormat); // Uniq ID created for the Data Format
using (var outputTopic = client.OpenOutputTopic(topicName)) // Open a KafkaOutputTopic
{
var output = new SessionTelemetryDataOutput(outputTopic, dataFormatId, dataFormatClient);
output.SessionOutput.AddSessionDependency(DependencyTypes.DataFormat, dataFormatId); // Add session dependencies to the output
output.SessionOutput.AddSessionDependency(DependencyTypes.AtlasConfiguration, atlasConfigurationId);
output.SessionOutput.SessionState = StreamSessionState.Open; // set the sessions state to open
output.SessionOutput.SessionStart = DateTime.Now; // set the session start to current time
output.SessionOutput.SessionIdentifier = "events_" + DateTime.Now; // set a custom session identifier
output.SessionOutput.SendSession();
var events = GenerateEvents(20, (DateTime)output.SessionOutput.SessionStart); // Generate some events data
var tasks = events.Select(ev => output.EventsOutput.SendEvent(ev)).ToArray(); // enqueue and send the events to the output through the EventsOutput
Task.WaitAll(tasks);
output.SessionOutput.SessionState = StreamSessionState.Closed; // set session state to closed. In case of any unintended session close, set state to Truncated
output.SessionOutput.SendSession(); // send session
}
}
/// <summary>
/// This method creates stream pipeline for each child stream.
/// </summary>
/// <param name="dataFormatClient">The data format client.</param>
/// <param name="outputTopic">The output topic.</param>
/// <param name="streamId">The stream identifier.</param>
/// <returns>Stream pipeline.</returns>
private IStreamInput CreateStreamInput(DataFormatClient dataFormatClient, IOutputTopic outputTopic, string streamId)
{
// these templates provide commonly-combined data, but you can make your own
var input = new SessionTelemetryDataInput(streamId, dataFormatClient);
var output = new SessionTelemetryDataOutput(outputTopic, dataFormatId, dataFormatClient);
// data is split into named feeds; use default feed if you don't need that
var inputFeed = input.DataInput.BindFeed(InputFeed, CreateInputDataFormat());
var outputFeed = output.DataOutput.BindFeed(OutputFeed);
output.SessionOutput.AddSessionDependency(DependencyTypes.AtlasConfiguration, atlasConfId);
output.SessionOutput.AddSessionDependency(DependencyTypes.DataFormat, dataFormatId);
// automatically propagate session metadata and lifecycle
input.LinkToOutput(output.SessionOutput, identifier => identifier + "_" + OutputTopicName);
input.LapsInput.LapStarted += (s, e) => output.LapsOutput.SendLap(e.Lap);
// react to data
inputFeed.DataBuffered += (sender, e) =>
{
var data = ProcessData(e.Buffer);
outputFeed.EnqueueAndSendData(data);
};
return(input);
}
public IStreamInput CreateStreamInput(string streamId)
{
Console.WriteLine($"Stream {streamId} started.");
// these templates provide commonly-combined data, but you can make your own
var input = new SessionTelemetryDataInput(streamId, dataFormatClient);
var output = new SessionTelemetryDataOutput(outputTopic, this.outputDataFormatId, dataFormatClient);
this.outputFeed = output.DataOutput.BindFeed("");
// we add output format reference to output session.
output.SessionOutput.AddSessionDependency(DependencyTypes.DataFormat, this.outputDataFormatId);
output.SessionOutput.AddSessionDependency(DependencyTypes.AtlasConfiguration, this.outputAtlasConfId);
// automatically propagate session metadata and lifecycle
input.LinkToOutput(output.SessionOutput, identifier => identifier + "_Models");
// we simply forward laps.
input.LapsInput.LapStarted += (s, e) => output.LapsOutput.SendLap(e.Lap);
// we bind our models to specific feed and parameters.
input.DataInput.BindDefaultFeed("gLat:Chassis", "gLong:Chassis").DataBuffered += this.gTotalModel;
input.StreamFinished += (s, e) => Console.WriteLine($"Stream {e.StreamId} ended.");
return(input);
}
private void GenerateData(IOutputTopic topic, DataFormatClient dataFormatClient, string dataFormatId, string atlasConfigurationId)
{
var foregroundColor = (ConsoleColor)(15 - this.topicLists.IndexOf(topic.TopicName));
const int steps = 120000;
const int delay = (int)(1000 / Frequency);
var start = DateTime.UtcNow;
var epoch = start.ToTelemetryTime();
var output = new SessionTelemetryDataOutput(topic, dataFormatId, dataFormatClient);
var outputFeed = output.DataOutput.BindDefaultFeed();
// declare a session
output.SessionOutput.AddSessionDependency(DependencyTypes.DataFormat, dataFormatId);
output.SessionOutput.AddSessionDependency(DependencyTypes.AtlasConfiguration, atlasConfigurationId);
output.SessionOutput.SessionState = StreamSessionState.Open;
output.SessionOutput.SessionStart = start;
output.SessionOutput.SessionIdentifier = "random_walk" + DateTime.Now.TimeOfDay;
output.SessionOutput.SendSession();
// generate data points
var data = outputFeed.MakeTelemetryData(1, epoch);
var sinParam = data.Parameters[0];
var cosParam = data.Parameters[1];
for (var step = 1; step <= steps; step++)
{
Thread.Sleep(delay);
// writing a single data point for simplicity, but you can send chunks of data
// timestamps expressed in ns since the epoch (which is the start of the session)
var elapsedNs = step * delay * 1000000L;
data.TimestampsNanos[0] = elapsedNs;
sinParam.Statuses[0] = DataStatus.Sample;
sinParam.AvgValues[0] = Math.Sin(step / 50.0);
cosParam.Statuses[0] = DataStatus.Sample;
cosParam.AvgValues[0] = Math.Cos(step / 100.0);
output.SessionOutput.SessionDurationNanos = elapsedNs;
outputFeed.EnqueueAndSendData(data);
Console.ForegroundColor = foregroundColor;
Console.Write(">");
if (Console.KeyAvailable && Console.ReadKey(true).Key == ConsoleKey.Escape)
{
break;
}
}
output.SessionOutput.SessionState = StreamSessionState.Closed;
output.SessionOutput.SendSession();
}
private void GenerateData(IOutputTopic topic, DataFormatClient dataFormatClient, string dataFormatId, string atlasConfigurationId)
{
const int steps = 12000;
const int delay = (int)(1000 / Frequency);
var random = new Random();
var start = DateTime.UtcNow;
var epoch = start.ToTelemetryTime();
var output = new SessionTelemetryDataOutput(topic, dataFormatId, dataFormatClient);
var outputFeed = output.DataOutput.BindDefaultFeed();
// declare a session
output.SessionOutput.AddSessionDependency(DependencyTypes.DataFormat, dataFormatId);
output.SessionOutput.AddSessionDependency(DependencyTypes.AtlasConfiguration, atlasConfigurationId);
output.SessionOutput.SessionState = StreamSessionState.Open;
output.SessionOutput.SessionStart = start.Date;
output.SessionOutput.SessionIdentifier = "random_walk";
output.SessionOutput.SendSession();
// generate data points
var data = outputFeed.MakeTelemetryData(1000000, epoch);
var param = data.Parameters[0];
var rangeWalker = new RandomRangeWalker(0, 1);
for (var step = 1; step <= steps; step++)
{
Thread.Sleep(delay);
for (var i = 0; i < 1000000; i++)
{
// writing a single data point for simplicity, but you can send chunks of data
// timestamps expressed in ns since the epoch (which is the start of the session)
var elapsedNs = step * delay * 1000000L;
data.TimestampsNanos[i] = elapsedNs;
param.Statuses[i] = DataStatus.Sample;
var value = rangeWalker.GetNext();
param.AvgValues[i] = value;
output.SessionOutput.SessionDurationNanos = elapsedNs;
//Logger.Info(NumberToBarString.Convert(value));
}
outputFeed.EnqueueAndSendData(data);
}
output.SessionOutput.SessionState = StreamSessionState.Closed;
output.SessionOutput.SendSession();
}
public void WriteTData()
{
const string brokerList = "localhost:9092"; // The host and port where the Kafka broker is running
const string groupName = "dev"; // The group name
const string topicName = "data_in"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the data_in_announce
var dependencyServiceUri = new Uri("http://localhost:8180/api/dependencies/"); // The URI where the dependency services are running
var client = new KafkaStreamClient(brokerList); // Create a new KafkaStreamClient for connecting to Kafka broker
var dataFormatClient = new DataFormatClient(new HttpDependencyClient(dependencyServiceUri, groupName)); // Create a new DataFormatClient
var httpDependencyClient = new HttpDependencyClient(dependencyServiceUri, groupName); // DependencyClient stores the Data format, Atlas Configuration
var atlasConfigurationId = new AtlasConfigurationClient(httpDependencyClient).PutAndIdentifyAtlasConfiguration(AtlasConfiguration); // Uniq ID created for the AtlasConfiguration
var dataFormat = DataFormat.DefineFeed().Parameter(ParameterId).BuildFormat(); // Create a dataformat based on the parameters, using the parameter id
var dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(dataFormat); // Uniq ID created for the Data Format
using (var outputTopic = client.OpenOutputTopic(topicName)) // Open a KafkaOutputTopic
{
const int sampleCount = 10000;
var output = new SessionTelemetryDataOutput(outputTopic, dataFormatId, dataFormatClient);
output.SessionOutput.AddSessionDependency(DependencyTypes.DataFormat, dataFormatId); // Add session dependencies to the output
output.SessionOutput.AddSessionDependency(DependencyTypes.AtlasConfiguration, atlasConfigurationId);
output.SessionOutput.SessionState = StreamSessionState.Open; // set the sessions state to open
output.SessionOutput.SessionStart = DateTime.Now; // set the session start to current time
output.SessionOutput.SessionDurationNanos = sampleCount * Interval; // duration should be time elapsed between session start time and last sample time
output.SessionOutput.SessionIdentifier = "data_" + DateTime.Now; // set a custom session identifier
output.SessionOutput.SendSession();
var telemetryData = GenerateData(sampleCount, (DateTime)output.SessionOutput.SessionStart); // Generate some telemetry data
const string feedName = ""; // As sample DataFormat uses default feed, we will leave this empty.
var outputFeed = output.DataOutput.BindFeed(feedName); // bind your feed by its name to the Data Output
Task.WaitAll(outputFeed.EnqueueAndSendData(telemetryData)); // enqueue and send the data to the output through the outputFeed
output.SessionOutput.SessionState = StreamSessionState.Closed; // set session state to closed. In case of any unintended session close, set state to Truncated
output.SessionOutput.SendSession(); // send session
}
}
public Writer(Uri dependencyServiceUri, AtlasConfiguration atlasConfiguration, DataFormat dataFormat,
string group, IOutputTopic topic, bool enableCache = true)
{
var httpDependencyClient =
new HttpDependencyClient(dependencyServiceUri, group,
enableCache); // DependencyClient stores the Data format, Atlas Configuration
var dataFormatClient = new DataFormatClient(httpDependencyClient);
var atlasConfigurationClient = new AtlasConfigurationClient(httpDependencyClient);
var atlasConfigurationId =
atlasConfigurationClient
.PutAndIdentifyAtlasConfiguration(atlasConfiguration); // Uniq ID created for the AtlasConfiguration
var dataFormatId =
dataFormatClient.PutAndIdentifyDataFormat(dataFormat); // Uniq ID created for the Data Format
TopicName = topic.TopicName;
//Init Session
session = new SessionTelemetryDataOutput(topic, dataFormatId, dataFormatClient);
session.SessionOutput.AddSessionDependency(DependencyTypes.DataFormat, dataFormatId);
session.SessionOutput.AddSessionDependency(DependencyTypes.AtlasConfiguration, atlasConfigurationId);
SessionOutput = session.SessionOutput;
}
