public void WriteEvents()
{
const string brokerList = "localhost:9092"; // The host and port where the Kafka broker is running
const string groupName = "dev"; // The dependency group name
const string topicName = "events_sample"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the data_in_announce
var dependencyServiceUri = new Uri("http://localhost:8180/api/dependencies/"); // The URI where the dependency services are running
var client = new KafkaStreamClient(brokerList); // Create a new KafkaStreamClient for connecting to Kafka broker
var dataFormatClient = new DataFormatClient(new HttpDependencyClient(dependencyServiceUri, groupName)); // Create a new DataFormatClient
var httpDependencyClient = new HttpDependencyClient(dependencyServiceUri, groupName); // DependencyClient stores the Data format, Atlas Configuration
var atlasConfigurationId = new AtlasConfigurationClient(httpDependencyClient).PutAndIdentifyAtlasConfiguration(AtlasConfiguration); // Uniq ID created for the AtlasConfiguration
var dataFormat = DataFormat.DefineFeed().BuildFormat(); // Create a dataformat based on the parameters, using the parameter id
var dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(dataFormat); // Uniq ID created for the Data Format
using (var outputTopic = client.OpenOutputTopic(topicName)) // Open a KafkaOutputTopic
{
var output = new SessionTelemetryDataOutput(outputTopic, dataFormatId, dataFormatClient);
output.SessionOutput.AddSessionDependency(DependencyTypes.DataFormat, dataFormatId); // Add session dependencies to the output
output.SessionOutput.AddSessionDependency(DependencyTypes.AtlasConfiguration, atlasConfigurationId);
output.SessionOutput.SessionState = StreamSessionState.Open; // set the sessions state to open
output.SessionOutput.SessionStart = DateTime.Now; // set the session start to current time
output.SessionOutput.SessionIdentifier = "events_" + DateTime.Now; // set a custom session identifier
output.SessionOutput.SendSession();
var events = GenerateEvents(20, (DateTime)output.SessionOutput.SessionStart); // Generate some events data
var tasks = events.Select(ev => output.EventsOutput.SendEvent(ev)).ToArray(); // enqueue and send the events to the output through the EventsOutput
Task.WaitAll(tasks);
output.SessionOutput.SessionState = StreamSessionState.Closed; // set session state to closed. In case of any unintended session close, set state to Truncated
output.SessionOutput.SendSession(); // send session
}
}
/// <summary>
/// Define source data format of vCar parameter we using here.
/// </summary>
protected override DataFormat CreateInputDataFormat()
{
return(DataFormat.DefineFeed()
.Parameter("vCar:Chassis")
.AtFrequency(Frequency)
.BuildFormat());
}
public void Run()
{
var dependenciesClient = new HttpDependencyClient(DependenciesUri, "dev");
var dataFormatClient = new DataFormatClient(dependenciesClient);
var atlasConfigurationClient = new AtlasConfigurationClient(dependenciesClient);
var dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(
DataFormat.DefineFeed().Parameters(new List <string> {
"Sin(x)", "Cos(x)"
}).AtFrequency(Frequency).BuildFormat());
var atlasConfigurationId = atlasConfigurationClient.PutAndIdentifyAtlasConfiguration(MakeAtlasConfiguration());
var tasks = new List <Task>();
using (var client = new KafkaStreamClient(brokerList))
{
foreach (var topicName in topicLists)
{
tasks.Add(Task.Factory.StartNew(() =>
{
using (var topic = client.OpenOutputTopic(topicName))
{
GenerateData(topic, dataFormatClient, dataFormatId, atlasConfigurationId);
Console.WriteLine("Hit <enter> to exit");
Console.ReadLine();
}
}));
}
Task.WaitAll(tasks.ToArray());
}
}
/// <summary>
/// Define source data format of vCar parameter we using here.
/// </summary>
protected override DataFormat CreateInputDataFormat()
{
return(DataFormat.DefineFeed()
.Parameters(new List <string> {
"Sin(x)", "Cos(x)"
})
.AtFrequency(Frequency)
.BuildFormat());
}
/// <summary>
/// Defines double vCar parameter data format.
/// </summary>
protected override DataFormat CreateOutputDataFormat()
{
return(DataFormat.DefineFeed()
.Parameters(new List <string> {
"Abs(Sin(x)+Cos(2x))"
})
.AtFrequency(Frequency)
.BuildFormat());
}
/// <summary>
/// Creates a DataFormat object from the default feed and the given frequency and parameterId.
/// </summary>
/// <returns>The Dataformat object built by the configured DataformatBuilder.</returns>
public static DataFormat GetDataFormat()
{
// Data formats can contain multiple feeds, each feed has a frequency, which is 100 by default.
// Name is optional, but if you wish to define multiple feeds, give them names
// else previous unnamed feed will be overwritten. The default is the empty string.
var dataFormatBuilder = DataFormat.DefineFeed();
// Setting frequency to the previously started feed. This can be only one value, if you set
// it multiple times, previous value will be overwritten. If you wish to use multiple
// frequencies, use new feed by calling .NextFeed()
dataFormatBuilder.AtFrequency(Frequency);
// Adding parameter to the previously started feed by its parameterId.
dataFormatBuilder.Parameter(ParameterId);
var dataFormat = dataFormatBuilder.BuildFormat();
return(dataFormat);
}
/// <summary>
/// Creates a DataFormat object from the default feed and the given frequency and sample parameterId.
/// </summary>
/// <returns>The Dataformat object built by the configured DataformatBuilder.</returns>
public static DataFormat GetDataFormat()
{
// Data formats can contain multiple feeds, each feed has a frequency, which is 100 by default.
// Name is optional, but if you wish to define multiple feeds, give them names
// else previous unnamed feed will be overwritten. The default is the empty string.
var dataFormatBuilder = DataFormat.DefineFeed();
// Setting frequency to the previously started feed. This can be only one value, if you set
// it multiple times, previous value will be overwritten. If you wish to use multiple
// frequencies, use new feed by calling .NextFeed()
dataFormatBuilder.AtFrequency(Frequency);
// VERY IMPORTANT information:
// The order you add will define which index you'll have to use when adding data to the telemetry data
// in this case vCar will have index 0,NGear will have index 1
dataFormatBuilder.Parameters(VcarParameterId, GearParameterId); // Adding parameters to the previously started feed.
var dataFormat = dataFormatBuilder.BuildFormat();
return(dataFormat);
}
public void Run()
{
var dependenciesClient = new HttpDependencyClient(DependencyServiceConfig.Uri, "dev");
var dataFormatClient = new DataFormatClient(dependenciesClient);
var atlasConfigurationClient = new AtlasConfigurationClient(dependenciesClient);
var dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(
DataFormat.DefineFeed().Parameter(TopicConfiguration.ParameterIdentifier).AtFrequency(Frequency).BuildFormat());
var atlasConfigurationId = atlasConfigurationClient.PutAndIdentifyAtlasConfiguration(MakeAtlasConfiguration());
using (var outputTopicWrapper = CreateOutputTopicWrapper())
{
Console.WriteLine("Hit <enter> to start generating data");
Console.ReadLine();
GenerateData(outputTopicWrapper.Topic, dataFormatClient, dataFormatId, atlasConfigurationId);
Console.WriteLine();
Console.WriteLine("Hit <enter> to exit");
Console.ReadLine();
}
}
public void WriteTData()
{
const string brokerList = "localhost:9092"; // The host and port where the Kafka broker is running
const string groupName = "dev"; // The group name
const string topicName = "data_in"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the data_in_announce
var dependencyServiceUri = new Uri("http://localhost:8180/api/dependencies/"); // The URI where the dependency services are running
var client = new KafkaStreamClient(brokerList); // Create a new KafkaStreamClient for connecting to Kafka broker
var dataFormatClient = new DataFormatClient(new HttpDependencyClient(dependencyServiceUri, groupName)); // Create a new DataFormatClient
var httpDependencyClient = new HttpDependencyClient(dependencyServiceUri, groupName); // DependencyClient stores the Data format, Atlas Configuration
var atlasConfigurationId = new AtlasConfigurationClient(httpDependencyClient).PutAndIdentifyAtlasConfiguration(AtlasConfiguration); // Uniq ID created for the AtlasConfiguration
var dataFormat = DataFormat.DefineFeed().Parameter(ParameterId).BuildFormat(); // Create a dataformat based on the parameters, using the parameter id
var dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(dataFormat); // Uniq ID created for the Data Format
using (var outputTopic = client.OpenOutputTopic(topicName)) // Open a KafkaOutputTopic
{
const int sampleCount = 10000;
var output = new SessionTelemetryDataOutput(outputTopic, dataFormatId, dataFormatClient);
output.SessionOutput.AddSessionDependency(DependencyTypes.DataFormat, dataFormatId); // Add session dependencies to the output
output.SessionOutput.AddSessionDependency(DependencyTypes.AtlasConfiguration, atlasConfigurationId);
output.SessionOutput.SessionState = StreamSessionState.Open; // set the sessions state to open
output.SessionOutput.SessionStart = DateTime.Now; // set the session start to current time
output.SessionOutput.SessionDurationNanos = sampleCount * Interval; // duration should be time elapsed between session start time and last sample time
output.SessionOutput.SessionIdentifier = "data_" + DateTime.Now; // set a custom session identifier
output.SessionOutput.SendSession();
var telemetryData = GenerateData(sampleCount, (DateTime)output.SessionOutput.SessionStart); // Generate some telemetry data
const string feedName = ""; // As sample DataFormat uses default feed, we will leave this empty.
var outputFeed = output.DataOutput.BindFeed(feedName); // bind your feed by its name to the Data Output
Task.WaitAll(outputFeed.EnqueueAndSendData(telemetryData)); // enqueue and send the data to the output through the outputFeed
output.SessionOutput.SessionState = StreamSessionState.Closed; // set session state to closed. In case of any unintended session close, set state to Truncated
output.SessionOutput.SendSession(); // send session
}
}
//DRS wing open. Raw data in samples. timestamps are completely random, not periodic, 0Hz
/// <summary>
/// Creates a DataFormat object with the provided feed names, frequencies and parameterIds (Tuples)
/// </summary>
/// <param name="feedNameAndFrequencyAndParameterList">Feed name, frequncy and parameterId pairs used for DataFormat creation.</param>
/// <returns>The Dataformat object built by the configured DataformatBuilder.</returns>
public static DataFormat GetDataFormat(List <Tuple <string, double, string> > feedNameAndFrequencyAndParameterList)
{
// Data formats can contain multiple feeds, each feed has a frequency, which is 100 by default.
// Name is optional, but if you wish to define multiple feeds, give them names
// else previous unnamed feed will be overwritten. The default is the empty string.
var dataFormatBuilder = DataFormat.DefineFeed(feedNameAndFrequencyAndParameterList.First().Item1);// Use the first feed name for creating the DataFormatBuilder.
// It will be overwritten in the foreach loop
var parameterIds = GetParameterIds();
foreach (var feedNameFrequencyAndParameter in feedNameAndFrequencyAndParameterList)
{
var feedName = feedNameFrequencyAndParameter.Item1; // Item1 of the feedNameFrequencyAndParameter is used as the feed name
var frequency = feedNameFrequencyAndParameter.Item2; // Item2 of the feedNameFrequencyAndParameter is used as the frequency
var parameterId = feedNameFrequencyAndParameter.Item3; // Item3 of the feedNameFrequencyAndParameter is used as the parameter
dataFormatBuilder.NextFeed(feedName);
// Setting frequency to the previously started feed. This can be only one value, if you set
// it multiple times, previous value will be overwritten. If you wish to use multiple
// frequencies, use new feed by calling .NextFeed
dataFormatBuilder.AtFrequency(frequency);
if (!parameterIds.Contains(parameterId)) // Check if the provided parameter is in the sample Atlas configuration
{
throw new ArgumentException($"Parameter id \"{parameterId}\" is invalid!");
}
// VERY IMPORTANT information:
// The order you add will define which index you'll have to use when adding data to the telemetry data
// in this case Sin(x) will have index 0, Cos(x) will have index 1
dataFormatBuilder.Parameter(parameterId); // Adding parameters to the previously started feed.
}
var dataFormat = dataFormatBuilder.BuildFormat();
return(dataFormat);
}
public void Run(CancellationToken cancellationToken = default(CancellationToken))
{
var outputDataFormat = DataFormat.DefineFeed()
.Parameter("gTotal:vTag")
.AtFrequency(100)
.BuildFormat();
this.dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(outputDataFormat);
var atlasConfiguration = this.CreateAtlasConfiguration();
this.atlasConfId = this.acClient.PutAndIdentifyAtlasConfiguration(atlasConfiguration);
using (var client = new KafkaStreamClient(BrokerList))
using (var outputTopic = client.OpenOutputTopic(OutputTopicName))
using (var pipeline = client.StreamTopic(InputTopicName).Into(streamId => this.CreateStreamPipeline(streamId, outputTopic)))
{
cancellationToken.WaitHandle.WaitOne();
pipeline.Drain();
pipeline.WaitUntilStopped(TimeSpan.FromSeconds(1), CancellationToken.None);
}
}
/// <summary>
/// Creates a DataFormat object with the provided feed names and frequency pairs (Tuples) plus the sample parameterId
/// </summary>
/// <param name="feedNameAndFrequencyList">Feed name and frequncy pairs used for DataFormat creation.</param>
/// <returns>The Dataformat object built by the configured DataformatBuilder.</returns>
public static DataFormat GetDataFormat(List <Tuple <string, double> > feedNameAndFrequencyList)
{
// Data formats can contain multiple feeds, each feed has a frequency, which is 100 by default.
// Name is optional, but if you wish to define multiple feeds, give them names
// else previous unnamed feed will be overwritten. The default is the empty string.
var dataFormatBuilder = DataFormat.DefineFeed(feedNameAndFrequencyList.First().Item1); // Use the first feed name for creating the DataFormatBuilder.
// It will be overwritten in the foreach loop
foreach (var feedNameAndFrequency in feedNameAndFrequencyList)
{
dataFormatBuilder.NextFeed(feedNameAndFrequency.Item1); // Item1 of the feedNameAndFrequency is used as the feed name
// Setting frequency to the previously started feed. This can be only one value, if you set
// it multiple times, previous value will be overwritten. If you wish to use multiple
// frequencies, use new feed by calling .NextFeed
dataFormatBuilder.AtFrequency(feedNameAndFrequency.Item2); // Item2 of the feedNameAndFrequency is used as the frequency
dataFormatBuilder.Parameter(ParameterId); // Adding parameter to the previously started feed by its parameterId.
}
var dataFormat = dataFormatBuilder.BuildFormat();
return(dataFormat);
}
