/// <summary>
/// Listen input topic stream in a loop and produce new data into output topic.
/// </summary>
public Task Run(CancellationToken cancellationToken = default(CancellationToken))
{
return(Task.Factory.StartNew(() =>
{
// TODO: Replace this with dependency service REST API call.
var dependencyClient = new HttpDependencyClient(new Uri("http://localhost:8180/api/dependencies/"), "dev", false);
var dataFormatClient = new DataFormatClient(dependencyClient); // would be a web service in production
dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(CreateOutputDataFormat());
var acClient = new AtlasConfigurationClient(dependencyClient);
var atlasConfiguration = CreateAtlasConfiguration();
atlasConfId = acClient.PutAndIdentifyAtlasConfiguration(atlasConfiguration);
using (var client = new KafkaStreamClient(BrokerList))
{
client.ConsumerGroup = ConsumerGroup;
using (var enrichTopic = client.OpenOutputTopic(OutputTopicName))
using (var pipeline = CreateStreamPipeline(client, dataFormatClient, enrichTopic))
{
cancellationToken.WaitHandle.WaitOne();
pipeline.Drain();
pipeline.WaitUntilStopped(DrainTimeout, default(CancellationToken));
}
}
}));
}
public void WriteEvents()
{
const string brokerList = "localhost:9092"; // The host and port where the Kafka broker is running
const string groupName = "dev"; // The dependency group name
const string topicName = "events_sample"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the data_in_announce
var dependencyServiceUri = new Uri("http://localhost:8180/api/dependencies/"); // The URI where the dependency services are running
var client = new KafkaStreamClient(brokerList); // Create a new KafkaStreamClient for connecting to Kafka broker
var dataFormatClient = new DataFormatClient(new HttpDependencyClient(dependencyServiceUri, groupName)); // Create a new DataFormatClient
var httpDependencyClient = new HttpDependencyClient(dependencyServiceUri, groupName); // DependencyClient stores the Data format, Atlas Configuration
var atlasConfigurationId = new AtlasConfigurationClient(httpDependencyClient).PutAndIdentifyAtlasConfiguration(AtlasConfiguration); // Uniq ID created for the AtlasConfiguration
var dataFormat = DataFormat.DefineFeed().BuildFormat(); // Create a dataformat based on the parameters, using the parameter id
var dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(dataFormat); // Uniq ID created for the Data Format
using (var outputTopic = client.OpenOutputTopic(topicName)) // Open a KafkaOutputTopic
{
var output = new SessionTelemetryDataOutput(outputTopic, dataFormatId, dataFormatClient);
output.SessionOutput.AddSessionDependency(DependencyTypes.DataFormat, dataFormatId); // Add session dependencies to the output
output.SessionOutput.AddSessionDependency(DependencyTypes.AtlasConfiguration, atlasConfigurationId);
output.SessionOutput.SessionState = StreamSessionState.Open; // set the sessions state to open
output.SessionOutput.SessionStart = DateTime.Now; // set the session start to current time
output.SessionOutput.SessionIdentifier = "events_" + DateTime.Now; // set a custom session identifier
output.SessionOutput.SendSession();
var events = GenerateEvents(20, (DateTime)output.SessionOutput.SessionStart); // Generate some events data
var tasks = events.Select(ev => output.EventsOutput.SendEvent(ev)).ToArray(); // enqueue and send the events to the output through the EventsOutput
Task.WaitAll(tasks);
output.SessionOutput.SessionState = StreamSessionState.Closed; // set session state to closed. In case of any unintended session close, set state to Truncated
output.SessionOutput.SendSession(); // send session
}
}
public void Run()
{
var dependenciesClient = new HttpDependencyClient(DependenciesUri, "dev");
var dataFormatClient = new DataFormatClient(dependenciesClient);
var atlasConfigurationClient = new AtlasConfigurationClient(dependenciesClient);
var dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(
DataFormat.DefineFeed().Parameters(new List <string> {
"Sin(x)", "Cos(x)"
}).AtFrequency(Frequency).BuildFormat());
var atlasConfigurationId = atlasConfigurationClient.PutAndIdentifyAtlasConfiguration(MakeAtlasConfiguration());
var tasks = new List <Task>();
using (var client = new KafkaStreamClient(brokerList))
{
foreach (var topicName in topicLists)
{
tasks.Add(Task.Factory.StartNew(() =>
{
using (var topic = client.OpenOutputTopic(topicName))
{
GenerateData(topic, dataFormatClient, dataFormatId, atlasConfigurationId);
Console.WriteLine("Hit <enter> to exit");
Console.ReadLine();
}
}));
}
Task.WaitAll(tasks.ToArray());
}
}
public ModelSample()
{
this.dependencyClient = new HttpDependencyClient(new Uri(DependencyUrl), "dev", false);
this.dataFormatClient = new DataFormatClient(dependencyClient);
this.acClient = new AtlasConfigurationClient(dependencyClient);
}
/// <summary>
/// Reader constructor to set up the initial connections to Dependency Client, DataFormatClient and the
/// StreamPipelineBuilder for reading the stream
/// </summary>
/// <param name="dependencyServiceUri">The URI where the dependency service is running and accessible</param>
/// <param name="group">The environment specific group name</param>
/// <param name="pipelineBuilder">The pipeline builder for the stream to read from</param>
/// <param name="enableCache">Boolean flag to enable dependency caching in <see cref="HttpDependencyClient" /> </param>
public Reader(Uri dependencyServiceUri, string group, IStreamPipelineBuilder pipelineBuilder,
bool enableCache = true)
{
var dependenciesClient = new HttpDependencyClient(dependencyServiceUri, group, enableCache);
DataFormatClient = new DataFormatClient(dependenciesClient);
StreamPipelineBuilder = pipelineBuilder;
}
public void Run()
{
var dependenciesClient = new HttpDependencyClient(DependencyServiceConfig.Uri, "dev");
var dataFormatClient = new DataFormatClient(dependenciesClient);
var atlasConfigurationClient = new AtlasConfigurationClient(dependenciesClient);
var dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(
DataFormat.DefineFeed().Parameter(TopicConfiguration.ParameterIdentifier).AtFrequency(Frequency).BuildFormat());
var atlasConfigurationId = atlasConfigurationClient.PutAndIdentifyAtlasConfiguration(MakeAtlasConfiguration());
using (var outputTopicWrapper = CreateOutputTopicWrapper())
{
Console.WriteLine("Hit <enter> to start generating data");
Console.ReadLine();
GenerateData(outputTopicWrapper.Topic, dataFormatClient, dataFormatId, atlasConfigurationId);
Console.WriteLine();
Console.WriteLine("Hit <enter> to exit");
Console.ReadLine();
}
}
public void WriteTData()
{
const string brokerList = "localhost:9092"; // The host and port where the Kafka broker is running
const string groupName = "dev"; // The group name
const string topicName = "data_in"; // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the data_in_announce
var dependencyServiceUri = new Uri("http://localhost:8180/api/dependencies/"); // The URI where the dependency services are running
var client = new KafkaStreamClient(brokerList); // Create a new KafkaStreamClient for connecting to Kafka broker
var dataFormatClient = new DataFormatClient(new HttpDependencyClient(dependencyServiceUri, groupName)); // Create a new DataFormatClient
var httpDependencyClient = new HttpDependencyClient(dependencyServiceUri, groupName); // DependencyClient stores the Data format, Atlas Configuration
var atlasConfigurationId = new AtlasConfigurationClient(httpDependencyClient).PutAndIdentifyAtlasConfiguration(AtlasConfiguration); // Uniq ID created for the AtlasConfiguration
var dataFormat = DataFormat.DefineFeed().Parameter(ParameterId).BuildFormat(); // Create a dataformat based on the parameters, using the parameter id
var dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(dataFormat); // Uniq ID created for the Data Format
using (var outputTopic = client.OpenOutputTopic(topicName)) // Open a KafkaOutputTopic
{
const int sampleCount = 10000;
var output = new SessionTelemetryDataOutput(outputTopic, dataFormatId, dataFormatClient);
output.SessionOutput.AddSessionDependency(DependencyTypes.DataFormat, dataFormatId); // Add session dependencies to the output
output.SessionOutput.AddSessionDependency(DependencyTypes.AtlasConfiguration, atlasConfigurationId);
output.SessionOutput.SessionState = StreamSessionState.Open; // set the sessions state to open
output.SessionOutput.SessionStart = DateTime.Now; // set the session start to current time
output.SessionOutput.SessionDurationNanos = sampleCount * Interval; // duration should be time elapsed between session start time and last sample time
output.SessionOutput.SessionIdentifier = "data_" + DateTime.Now; // set a custom session identifier
output.SessionOutput.SendSession();
var telemetryData = GenerateData(sampleCount, (DateTime)output.SessionOutput.SessionStart); // Generate some telemetry data
const string feedName = ""; // As sample DataFormat uses default feed, we will leave this empty.
var outputFeed = output.DataOutput.BindFeed(feedName); // bind your feed by its name to the Data Output
Task.WaitAll(outputFeed.EnqueueAndSendData(telemetryData)); // enqueue and send the data to the output through the outputFeed
output.SessionOutput.SessionState = StreamSessionState.Closed; // set session state to closed. In case of any unintended session close, set state to Truncated
output.SessionOutput.SendSession(); // send session
}
}
public Writer(Uri dependencyServiceUri, AtlasConfiguration atlasConfiguration, DataFormat dataFormat,
string group, IOutputTopic topic, bool enableCache = true)
{
var httpDependencyClient =
new HttpDependencyClient(dependencyServiceUri, group,
enableCache); // DependencyClient stores the Data format, Atlas Configuration
var dataFormatClient = new DataFormatClient(httpDependencyClient);
var atlasConfigurationClient = new AtlasConfigurationClient(httpDependencyClient);
var atlasConfigurationId =
atlasConfigurationClient
.PutAndIdentifyAtlasConfiguration(atlasConfiguration); // Uniq ID created for the AtlasConfiguration
var dataFormatId =
dataFormatClient.PutAndIdentifyDataFormat(dataFormat); // Uniq ID created for the Data Format
TopicName = topic.TopicName;
//Init Session
session = new SessionTelemetryDataOutput(topic, dataFormatId, dataFormatClient);
session.SessionOutput.AddSessionDependency(DependencyTypes.DataFormat, dataFormatId);
session.SessionOutput.AddSessionDependency(DependencyTypes.AtlasConfiguration, atlasConfigurationId);
SessionOutput = session.SessionOutput;
}