Example #1
0
        public void WriteEvents()
        {
            const string brokerList           = "localhost:9092";                                                                               // The host and port where the Kafka broker is running
            const string groupName            = "dev";                                                                                          // The dependency group name
            const string topicName            = "events_sample";                                                                                // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the data_in_announce
            var          dependencyServiceUri = new Uri("http://localhost:8180/api/dependencies/");                                             // The URI where the dependency services are running

            var client               = new KafkaStreamClient(brokerList);                                                                       // Create a new KafkaStreamClient for connecting to Kafka broker
            var dataFormatClient     = new DataFormatClient(new HttpDependencyClient(dependencyServiceUri, groupName));                         // Create a new DataFormatClient
            var httpDependencyClient = new HttpDependencyClient(dependencyServiceUri, groupName);                                               // DependencyClient stores the Data format, Atlas Configuration

            var atlasConfigurationId = new AtlasConfigurationClient(httpDependencyClient).PutAndIdentifyAtlasConfiguration(AtlasConfiguration); // Uniq ID created for the AtlasConfiguration
            var dataFormat           = DataFormat.DefineFeed().BuildFormat();                                                                   // Create a dataformat based on the parameters, using the parameter id
            var dataFormatId         = dataFormatClient.PutAndIdentifyDataFormat(dataFormat);                                                   // Uniq ID created for the Data Format

            using (var outputTopic = client.OpenOutputTopic(topicName))                                                                         // Open a KafkaOutputTopic
            {
                var output = new SessionTelemetryDataOutput(outputTopic, dataFormatId, dataFormatClient);
                output.SessionOutput.AddSessionDependency(DependencyTypes.DataFormat, dataFormatId); // Add session dependencies to the output
                output.SessionOutput.AddSessionDependency(DependencyTypes.AtlasConfiguration, atlasConfigurationId);

                output.SessionOutput.SessionState      = StreamSessionState.Open;  // set the sessions state to open
                output.SessionOutput.SessionStart      = DateTime.Now;             // set the session start to current time
                output.SessionOutput.SessionIdentifier = "events_" + DateTime.Now; // set a custom session identifier
                output.SessionOutput.SendSession();

                var events = GenerateEvents(20, (DateTime)output.SessionOutput.SessionStart);  // Generate some events data
                var tasks  = events.Select(ev => output.EventsOutput.SendEvent(ev)).ToArray(); // enqueue and send the events to the output through the EventsOutput
                Task.WaitAll(tasks);

                output.SessionOutput.SessionState = StreamSessionState.Closed; // set session state to closed. In case of any unintended session close, set state to Truncated
                output.SessionOutput.SendSession();                            // send session
            }
        }
 /// <summary>
 /// Define source data format of vCar parameter we using here.
 /// </summary>
 protected override DataFormat CreateInputDataFormat()
 {
     return(DataFormat.DefineFeed()
            .Parameter("vCar:Chassis")
            .AtFrequency(Frequency)
            .BuildFormat());
 }
        public void Run()
        {
            var dependenciesClient       = new HttpDependencyClient(DependenciesUri, "dev");
            var dataFormatClient         = new DataFormatClient(dependenciesClient);
            var atlasConfigurationClient = new AtlasConfigurationClient(dependenciesClient);

            var dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(
                DataFormat.DefineFeed().Parameters(new List <string> {
                "Sin(x)", "Cos(x)"
            }).AtFrequency(Frequency).BuildFormat());

            var atlasConfigurationId = atlasConfigurationClient.PutAndIdentifyAtlasConfiguration(MakeAtlasConfiguration());

            var tasks = new List <Task>();

            using (var client = new KafkaStreamClient(brokerList))
            {
                foreach (var topicName in topicLists)
                {
                    tasks.Add(Task.Factory.StartNew(() =>
                    {
                        using (var topic = client.OpenOutputTopic(topicName))
                        {
                            GenerateData(topic, dataFormatClient, dataFormatId, atlasConfigurationId);
                            Console.WriteLine("Hit <enter> to exit");
                            Console.ReadLine();
                        }
                    }));
                }

                Task.WaitAll(tasks.ToArray());
            }
        }
Example #4
0
 /// <summary>
 /// Define source data format of vCar parameter we using here.
 /// </summary>
 protected override DataFormat CreateInputDataFormat()
 {
     return(DataFormat.DefineFeed()
            .Parameters(new List <string> {
         "Sin(x)", "Cos(x)"
     })
            .AtFrequency(Frequency)
            .BuildFormat());
 }
 /// <summary>
 /// Defines double vCar parameter data format.
 /// </summary>
 protected override DataFormat CreateOutputDataFormat()
 {
     return(DataFormat.DefineFeed()
            .Parameters(new List <string> {
         "Abs(Sin(x)+Cos(2x))"
     })
            .AtFrequency(Frequency)
            .BuildFormat());
 }
        /// <summary>
        /// Creates a DataFormat object from the default feed and the given frequency and parameterId.
        /// </summary>
        /// <returns>The Dataformat object built by the configured DataformatBuilder.</returns>
        public static DataFormat GetDataFormat()
        {
            // Data formats can contain multiple feeds, each feed has a frequency, which is 100 by default.
            // Name is optional, but if you wish to define multiple feeds, give them names
            // else previous unnamed feed will be overwritten. The default is the empty string.
            var dataFormatBuilder = DataFormat.DefineFeed();

            // Setting frequency to the previously started feed. This can be only one value, if you set
            // it multiple times, previous value will be overwritten. If you wish to use multiple
            // frequencies, use new feed by calling .NextFeed()
            dataFormatBuilder.AtFrequency(Frequency);

            // Adding parameter to the previously started feed by its parameterId.
            dataFormatBuilder.Parameter(ParameterId);

            var dataFormat = dataFormatBuilder.BuildFormat();

            return(dataFormat);
        }
Example #7
0
        /// <summary>
        /// Creates a DataFormat object from the default feed and the given frequency and sample parameterId.
        /// </summary>
        /// <returns>The Dataformat object built by the configured DataformatBuilder.</returns>
        public static DataFormat GetDataFormat()
        {
            // Data formats can contain multiple feeds, each feed has a frequency, which is 100 by default.
            // Name is optional, but if you wish to define multiple feeds, give them names
            // else previous unnamed feed will be overwritten. The default is the empty string.
            var dataFormatBuilder = DataFormat.DefineFeed();

            // Setting frequency to the previously started feed. This can be only one value, if you set
            // it multiple times, previous value will be overwritten. If you wish to use multiple
            // frequencies, use new feed by calling .NextFeed()
            dataFormatBuilder.AtFrequency(Frequency);

            // VERY IMPORTANT information:
            // The order you add will define which index you'll have to use when adding data to the telemetry data
            // in this case vCar will have index 0,NGear will have index 1
            dataFormatBuilder.Parameters(VcarParameterId, GearParameterId); // Adding parameters to the previously started feed.

            var dataFormat = dataFormatBuilder.BuildFormat();

            return(dataFormat);
        }
        public void Run()
        {
            var dependenciesClient       = new HttpDependencyClient(DependencyServiceConfig.Uri, "dev");
            var dataFormatClient         = new DataFormatClient(dependenciesClient);
            var atlasConfigurationClient = new AtlasConfigurationClient(dependenciesClient);

            var dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(
                DataFormat.DefineFeed().Parameter(TopicConfiguration.ParameterIdentifier).AtFrequency(Frequency).BuildFormat());

            var atlasConfigurationId = atlasConfigurationClient.PutAndIdentifyAtlasConfiguration(MakeAtlasConfiguration());

            using (var outputTopicWrapper = CreateOutputTopicWrapper())
            {
                Console.WriteLine("Hit <enter> to start generating data");
                Console.ReadLine();
                GenerateData(outputTopicWrapper.Topic, dataFormatClient, dataFormatId, atlasConfigurationId);
                Console.WriteLine();
                Console.WriteLine("Hit <enter> to exit");
                Console.ReadLine();
            }
        }
Example #9
0
        public void WriteTData()
        {
            const string brokerList           = "localhost:9092";                                                                               // The host and port where the Kafka broker is running
            const string groupName            = "dev";                                                                                          // The group name
            const string topicName            = "data_in";                                                                                      // The existing topic's name in the Kafka broker. The *_announce topic name must exist too. In this case the data_in_announce
            var          dependencyServiceUri = new Uri("http://localhost:8180/api/dependencies/");                                             // The URI where the dependency services are running

            var client               = new KafkaStreamClient(brokerList);                                                                       // Create a new KafkaStreamClient for connecting to Kafka broker
            var dataFormatClient     = new DataFormatClient(new HttpDependencyClient(dependencyServiceUri, groupName));                         // Create a new DataFormatClient
            var httpDependencyClient = new HttpDependencyClient(dependencyServiceUri, groupName);                                               // DependencyClient stores the Data format, Atlas Configuration

            var atlasConfigurationId = new AtlasConfigurationClient(httpDependencyClient).PutAndIdentifyAtlasConfiguration(AtlasConfiguration); // Uniq ID created for the AtlasConfiguration
            var dataFormat           = DataFormat.DefineFeed().Parameter(ParameterId).BuildFormat();                                            // Create a dataformat based on the parameters, using the parameter id
            var dataFormatId         = dataFormatClient.PutAndIdentifyDataFormat(dataFormat);                                                   // Uniq ID created for the Data Format

            using (var outputTopic = client.OpenOutputTopic(topicName))                                                                         // Open a KafkaOutputTopic
            {
                const int sampleCount = 10000;
                var       output      = new SessionTelemetryDataOutput(outputTopic, dataFormatId, dataFormatClient);
                output.SessionOutput.AddSessionDependency(DependencyTypes.DataFormat, dataFormatId); // Add session dependencies to the output
                output.SessionOutput.AddSessionDependency(DependencyTypes.AtlasConfiguration, atlasConfigurationId);

                output.SessionOutput.SessionState         = StreamSessionState.Open; // set the sessions state to open
                output.SessionOutput.SessionStart         = DateTime.Now;            // set the session start to current time
                output.SessionOutput.SessionDurationNanos = sampleCount * Interval;  // duration should be time elapsed between session start time and last sample time
                output.SessionOutput.SessionIdentifier    = "data_" + DateTime.Now;  // set a custom session identifier
                output.SessionOutput.SendSession();

                var telemetryData = GenerateData(sampleCount, (DateTime)output.SessionOutput.SessionStart); // Generate some telemetry data

                const string feedName   = "";                                                               // As sample DataFormat uses default feed, we will leave this empty.
                var          outputFeed = output.DataOutput.BindFeed(feedName);                             // bind your feed by its name to the Data Output

                Task.WaitAll(outputFeed.EnqueueAndSendData(telemetryData));                                 // enqueue and send the data to the output through the outputFeed

                output.SessionOutput.SessionState = StreamSessionState.Closed;                              // set session state to closed. In case of any unintended session close, set state to Truncated
                output.SessionOutput.SendSession();                                                         // send session
            }
        }
        //DRS wing open. Raw data in samples. timestamps are completely random, not periodic, 0Hz

        /// <summary>
        /// Creates a DataFormat object with the provided feed names, frequencies and parameterIds (Tuples)
        /// </summary>
        /// <param name="feedNameAndFrequencyAndParameterList">Feed name, frequncy and parameterId pairs used for DataFormat creation.</param>
        /// <returns>The Dataformat object built by the configured DataformatBuilder.</returns>
        public static DataFormat GetDataFormat(List <Tuple <string, double, string> > feedNameAndFrequencyAndParameterList)
        {
            // Data formats can contain multiple feeds, each feed has a frequency, which is 100 by default.
            // Name is optional, but if you wish to define multiple feeds, give them names
            // else previous unnamed feed will be overwritten. The default is the empty string.
            var dataFormatBuilder = DataFormat.DefineFeed(feedNameAndFrequencyAndParameterList.First().Item1);// Use the first feed name for creating the DataFormatBuilder.
            // It will be overwritten in the foreach loop

            var parameterIds = GetParameterIds();

            foreach (var feedNameFrequencyAndParameter in feedNameAndFrequencyAndParameterList)
            {
                var feedName    = feedNameFrequencyAndParameter.Item1; // Item1 of the feedNameFrequencyAndParameter is used as the feed name
                var frequency   = feedNameFrequencyAndParameter.Item2; // Item2 of the feedNameFrequencyAndParameter is used as the frequency
                var parameterId = feedNameFrequencyAndParameter.Item3; // Item3 of the feedNameFrequencyAndParameter is used as the parameter

                dataFormatBuilder.NextFeed(feedName);

                // Setting frequency to the previously started feed. This can be only one value, if you set
                // it multiple times, previous value will be overwritten. If you wish to use multiple
                // frequencies, use new feed by calling .NextFeed
                dataFormatBuilder.AtFrequency(frequency);


                if (!parameterIds.Contains(parameterId)) // Check if the provided parameter is in the sample Atlas configuration
                {
                    throw new ArgumentException($"Parameter id \"{parameterId}\" is invalid!");
                }

                // VERY IMPORTANT information:
                // The order you add will define which index you'll have to use when adding data to the telemetry data
                // in this case Sin(x) will have index 0, Cos(x) will have index 1
                dataFormatBuilder.Parameter(parameterId); // Adding parameters to the previously started feed.
            }

            var dataFormat = dataFormatBuilder.BuildFormat();

            return(dataFormat);
        }
Example #11
0
        public void Run(CancellationToken cancellationToken = default(CancellationToken))
        {
            var outputDataFormat = DataFormat.DefineFeed()
                                   .Parameter("gTotal:vTag")
                                   .AtFrequency(100)
                                   .BuildFormat();

            this.dataFormatId = dataFormatClient.PutAndIdentifyDataFormat(outputDataFormat);


            var atlasConfiguration = this.CreateAtlasConfiguration();

            this.atlasConfId = this.acClient.PutAndIdentifyAtlasConfiguration(atlasConfiguration);

            using (var client = new KafkaStreamClient(BrokerList))
                using (var outputTopic = client.OpenOutputTopic(OutputTopicName))
                    using (var pipeline = client.StreamTopic(InputTopicName).Into(streamId => this.CreateStreamPipeline(streamId, outputTopic)))
                    {
                        cancellationToken.WaitHandle.WaitOne();
                        pipeline.Drain();
                        pipeline.WaitUntilStopped(TimeSpan.FromSeconds(1), CancellationToken.None);
                    }
        }
        /// <summary>
        /// Creates a DataFormat object with the provided feed names and frequency pairs (Tuples) plus the sample parameterId
        /// </summary>
        /// <param name="feedNameAndFrequencyList">Feed name and frequncy pairs used for DataFormat creation.</param>
        /// <returns>The Dataformat object built by the configured DataformatBuilder.</returns>
        public static DataFormat GetDataFormat(List <Tuple <string, double> > feedNameAndFrequencyList)
        {
            // Data formats can contain multiple feeds, each feed has a frequency, which is 100 by default.
            // Name is optional, but if you wish to define multiple feeds, give them names
            // else previous unnamed feed will be overwritten. The default is the empty string.
            var dataFormatBuilder = DataFormat.DefineFeed(feedNameAndFrequencyList.First().Item1); // Use the first feed name for creating the DataFormatBuilder.

            // It will be overwritten in the foreach loop
            foreach (var feedNameAndFrequency in feedNameAndFrequencyList)
            {
                dataFormatBuilder.NextFeed(feedNameAndFrequency.Item1); // Item1 of the feedNameAndFrequency is used as the feed name

                // Setting frequency to the previously started feed. This can be only one value, if you set
                // it multiple times, previous value will be overwritten. If you wish to use multiple
                // frequencies, use new feed by calling .NextFeed
                dataFormatBuilder.AtFrequency(feedNameAndFrequency.Item2); // Item2 of the feedNameAndFrequency is used as the frequency

                dataFormatBuilder.Parameter(ParameterId);                  // Adding parameter to the previously started feed by its parameterId.
            }

            var dataFormat = dataFormatBuilder.BuildFormat();

            return(dataFormat);
        }