Exemple #1
0
        /// <summary>
        /// This is the main entry point for your service instance.
        /// </summary>
        /// <param name="cancellationToken">Canceled when Service Fabric needs to shut down this service instance.</param>
        protected override async Task RunAsync(CancellationToken cancellationToken)
        {
            var configPackage = FabricRuntime.GetActivationContext().GetConfigurationPackageObject("Config");
            var inputConfig   = configPackage.Settings.Sections["InputConfig"];

            KeyVaultManager keyManager   = new KeyVaultManager();
            string          keyVaultName = inputConfig.Parameters["KeyVaultName"].Value;
            string          iotDeviceConnectionStringKeyVaultKeyName        = inputConfig.Parameters["IotDeviceConnectionStringKeyVaultKeyName"].Value;
            string          eventhubConnectionStringKeyVaultKeyName         = inputConfig.Parameters["EventhubConnectionStringKeyVaultKeyName"].Value;
            string          dataSchemaStorageAccountKeyValueKeyVaultKeyName = inputConfig.Parameters["DataSchemaStorageAccountKeyValueKeyVaultKeyName"].Value;
            string          iotDeviceConnectionString = (iotDeviceConnectionStringKeyVaultKeyName.Length > 0) ? await keyManager.GetSecretStringAsync(keyVaultName, iotDeviceConnectionStringKeyVaultKeyName) : "";

            string ehConnectionString = (eventhubConnectionStringKeyVaultKeyName.Length > 0) ? await keyManager.GetSecretStringAsync(keyVaultName, eventhubConnectionStringKeyVaultKeyName) : "";

            string dataSchemaStorageAccountName     = inputConfig.Parameters["DataSchemaStorageAccountName"].Value;
            string dataSchemaStorageAccountKeyValue = (dataSchemaStorageAccountKeyValueKeyVaultKeyName.Length > 0) ? await keyManager.GetSecretStringAsync(keyVaultName, dataSchemaStorageAccountKeyValueKeyVaultKeyName) : "";

            string        dataSchemaStorageContainerName = inputConfig.Parameters["DataSchemaStorageContainerName"].Value;
            List <string> dataSchemaPathsWithinContainer = Array.ConvertAll(inputConfig.Parameters["DataSchemaPathWithinContainer"].Value.Split(','), p => p.Trim()).ToList();

            KafkaConnection kafkaEnabledEventHubConn = new KafkaConnection
            {
                Topics           = (inputConfig.Parameters["KafkaTopics"].Value.Length > 0) ? Array.ConvertAll(inputConfig.Parameters["KafkaTopics"].Value.Split(','), p => p.Trim()).ToList() : new List <string>(),
                ConnectionString = (inputConfig.Parameters["KafkaConnectionStringKeyVaultKeyName"].Value.Length > 0) ? await keyManager.GetSecretStringAsync(keyVaultName, inputConfig.Parameters["KafkaConnectionStringKeyVaultKeyName"].Value) : ""
            };

            KafkaConnection kafkaHDInsightConn = new KafkaConnection
            {
                Topics           = (inputConfig.Parameters["KafkaHDInsightTopics"].Value.Length > 0) ? Array.ConvertAll(inputConfig.Parameters["KafkaHDInsightTopics"].Value.Split(','), p => p.Trim()).ToList() : new List <string>(),
                BootstrapServers = (inputConfig.Parameters["KafkaHDInsightBrokersKeyVaultKeyName"].Value.Length > 0) ? await keyManager.GetSecretStringAsync(keyVaultName, inputConfig.Parameters["KafkaHDInsightBrokersKeyVaultKeyName"].Value) : ""
            };

            if (!string.IsNullOrEmpty(kafkaEnabledEventHubConn.ConnectionString))
            {
                Regex regex = new Regex(@"sb?://([\w\d\.]+).*");
                kafkaEnabledEventHubConn.BootstrapServers = regex.Match(kafkaEnabledEventHubConn.ConnectionString).Groups[1].Value + ":9093";
                WebClient webClient = new WebClient();
                webClient.DownloadFile("https://curl.haxx.se/ca/cacert.pem", @".\cacert.pem");
            }

            List <DataSchema> dataSchemaList = new List <DataSchema>();

            foreach (var dataSchemaPathWithinContainer in dataSchemaPathsWithinContainer)
            {
                var dataSchemaFileContent = await GetDataSchemaAndRules(dataSchemaStorageAccountName, dataSchemaStorageAccountKeyValue, dataSchemaStorageContainerName, dataSchemaPathWithinContainer);

                dataSchemaFileContent.currentCounter = 1;
                dataSchemaList.Add(dataSchemaFileContent);
            }

            Stopwatch stopwatchDelay     = new Stopwatch();
            Stopwatch stopwatchThreshold = new Stopwatch();

            stopwatchThreshold.Start();
            DataGen dataGenInstance = new DataGen();

            while (true)
            {
                stopwatchDelay.Restart();
                cancellationToken.ThrowIfCancellationRequested();

                if (stopwatchThreshold.Elapsed.TotalMinutes >= 1440)
                {
                    stopwatchThreshold.Restart();
                }

                foreach (var dataSchemaFileContent in dataSchemaList)
                {
                    if (dataSchemaFileContent.currentCounter >= dataSchemaFileContent.rulesCounterRefreshInMinutes)
                    {
                        dataSchemaFileContent.currentCounter = 1;
                    }

                    List <JObject> dataStreams = new List <JObject>();
                    foreach (var ds in dataSchemaFileContent.dataSchema)
                    {
                        if (stopwatchThreshold.Elapsed.Minutes % ds.simulationPeriodInMinute == 0)
                        {
                            //generate random data
                            dataGenInstance.GenerateRandomData(dataStreams, ds);

                            //generate rules triggering data only for the 0th node in SF to avoid data duplication
                            if ((ds.rulesData != null) && (this.Context.NodeContext.NodeName.Substring(this.Context.NodeContext.NodeName.Length - 1) == "0"))
                            {
                                dataGenInstance.GenerateDataRules(dataStreams, ds, dataSchemaFileContent.currentCounter);
                            }
                        }
                    }

                    if (dataStreams.Count > 0)
                    {
                        await SendData(dataStreams, ehConnectionString, iotDeviceConnectionString, kafkaEnabledEventHubConn, kafkaHDInsightConn);
                    }
                    dataSchemaFileContent.currentCounter++;
                }

                var setDelay = ((60 - stopwatchDelay.Elapsed.TotalSeconds) > 0) ? (60 - stopwatchDelay.Elapsed.TotalSeconds) : 1;
                await Task.Delay(TimeSpan.FromSeconds(setDelay), cancellationToken);
            }
        }
Exemple #2
0
        /// <summary>
        /// This is the main entry point for your service instance.
        /// </summary>
        /// <param name="cancellationToken">Canceled when Service Fabric needs to shut down this service instance.</param>
        protected override async Task RunAsync(CancellationToken cancellationToken)
        {
            var configPackage = FabricRuntime.GetActivationContext().GetConfigurationPackageObject("Config");
            var inputConfig   = configPackage.Settings.Sections["InputConfig"];

            KeyVaultManager keyManager   = new KeyVaultManager();
            string          keyVaultName = inputConfig.Parameters["KeyVaultName"].Value;
            string          iotDeviceConnectionStringKeyVaultKeyName        = inputConfig.Parameters["IotDeviceConnectionStringKeyVaultKeyName"].Value;
            string          eventhubConnectionStringKeyVaultKeyName         = inputConfig.Parameters["EventhubConnectionStringKeyVaultKeyName"].Value;
            string          dataSchemaStorageAccountKeyValueKeyVaultKeyName = inputConfig.Parameters["DataSchemaStorageAccountKeyValueKeyVaultKeyName"].Value;
            string          iotDeviceConnectionString = (iotDeviceConnectionStringKeyVaultKeyName.Length > 0) ? await keyManager.GetSecretStringAsync(keyVaultName, iotDeviceConnectionStringKeyVaultKeyName) : "";

            string ehConnectionString = (eventhubConnectionStringKeyVaultKeyName.Length > 0) ? await keyManager.GetSecretStringAsync(keyVaultName, eventhubConnectionStringKeyVaultKeyName) : "";

            string dataSchemaStorageAccountName     = inputConfig.Parameters["DataSchemaStorageAccountName"].Value;
            string dataSchemaStorageAccountKeyValue = (dataSchemaStorageAccountKeyValueKeyVaultKeyName.Length > 0) ? await keyManager.GetSecretStringAsync(keyVaultName, dataSchemaStorageAccountKeyValueKeyVaultKeyName) : "";

            string dataSchemaStorageContainerName = inputConfig.Parameters["DataSchemaStorageContainerName"].Value;
            string dataSchemaPathWithinContainer  = inputConfig.Parameters["DataSchemaPathWithinContainer"].Value;

            var dataSchemaFileContent = await GetDataSchemaAndRules(dataSchemaStorageAccountName, dataSchemaStorageAccountKeyValue, dataSchemaStorageContainerName, dataSchemaPathWithinContainer);

            Stopwatch stopwatchDelay     = new Stopwatch();
            Stopwatch stopwatchThreshold = new Stopwatch();

            stopwatchThreshold.Start();
            DataGen dataGenInstance = new DataGen();

            while (true)
            {
                stopwatchDelay.Restart();
                cancellationToken.ThrowIfCancellationRequested();

                if (stopwatchThreshold.Elapsed.TotalMinutes >= 1440)
                {
                    stopwatchThreshold.Restart();
                }
                if (_counter >= dataSchemaFileContent.rulesCounterRefreshInMinutes)
                {
                    _counter = 1;
                }

                List <JObject> dataStreams = new List <JObject>();
                foreach (var ds in dataSchemaFileContent.dataSchema)
                {
                    if (stopwatchThreshold.Elapsed.Minutes % ds.simulationPeriodInMinute == 0)
                    {
                        //generate random data
                        dataGenInstance.GenerateRandomData(dataStreams, ds);

                        //generate rules triggering data only for the 0th node in SF to avoid data duplication
                        if ((ds.rulesData != null) && (this.Context.NodeContext.NodeName.Substring(this.Context.NodeContext.NodeName.Length - 1) == "0"))
                        {
                            dataGenInstance.GenerateDataRules(dataStreams, ds, _counter);
                        }
                    }
                }

                if (dataStreams.Count > 0)
                {
                    await SendData(ehConnectionString, iotDeviceConnectionString, dataStreams);
                }

                _counter++;
                var setDelay = ((60 - stopwatchDelay.Elapsed.TotalSeconds) > 0) ? (60 - stopwatchDelay.Elapsed.TotalSeconds) : 1;
                await Task.Delay(TimeSpan.FromSeconds(setDelay), cancellationToken);
            }
        }