Beispiel #1
0
        private static void ParseLocalConfigAndAndMineData(string apiDataMinerConfigFileLocation)
        {
            var databaseConnectionString = ConfigurationManager.AppSettings["SQLAzureDatabaseEntityFrameworkConnectionString"];
            var wattTimeApiKey           = ConfigurationManager.AppSettings["wattTimeApiKey"];
            var wundergroundApiKey       = ConfigurationManager.AppSettings["wundergroundApiKey"];
            var apiDataMiner             = new ApiDataMiner(databaseConnectionString);

            apiDataMiner.ParseMinerSettingsFileAndMineData(apiDataMinerConfigFileLocation, wattTimeApiKey, wundergroundApiKey);
        }
        private async Task RunAsync(CancellationToken cancellationToken)
        {
            Logger.Information("Data Miner Worker Role is Starting", "RunAsync()");

            var databaseConnectionString     = CloudConfigurationManager.GetSetting("SQLAzureDatabaseEntityFrameworkConnectionString");
            var azureStorageConnectionString = CloudConfigurationManager.GetSetting("AzureStorageConnectionString");
            var schedulerQueueName           = CloudConfigurationManager.GetSetting("SchedulerQueueName");

            var storageQueueManager = new AzureStorageQueueManager(azureStorageConnectionString, schedulerQueueName);

            while (!cancellationToken.IsCancellationRequested)
            {
                // Check if there are any messages on the queue to instruct the data miner what to do
                var message = storageQueueManager.GetMessage();
                if (message != null)
                {
                    // Instruction to run the data mining for the local config file
                    if (message.AsString.Contains("RunMinerForLocalConfigFile"))
                    {
                        // Parse config XML File defining regions, emissions regions, weather regions, etc.
                        using (
                            new TimedOperation(
                                "Beginning Mining of data pointed to by the configuration file",
                                "DataMinerWorkerRole.FullMinerCall"))
                        {
                            try
                            {
                                const string ConfigPath   = @".\ApiDataMinerConfigs\ApiDataMinerConfigs.xml";
                                var          apiDataMiner = new ApiDataMiner(databaseConnectionString);
                                apiDataMiner.ParseMinerSettingsFileAndMineData(ConfigPath);
                            }
                            catch (Exception exception)
                            {
                                Logger.Error(
                                    "DataMinerWorkerRole encountered an exception mining all data pointed to by the configuration file",
                                    "DataMinerWorkerRole.FullMinerCall",
                                    null,
                                    exception);
                            }
                        }
                    }

                    /* Add logic for any other mining here, and add a corresponding job to the Scheduler to send messages to the queue to kick it off */
                }

                // Sleep for a time before checking the queue again
                var numberOfMinutesToSleep = 1;
                Logger.Information($"Sleping for {numberOfMinutesToSleep} minutes", "RunAsync()");
                await Task.Delay(1000 * 60 *numberOfMinutesToSleep, cancellationToken);
            }
        }
        private static void ParseLocalConfigAndAndMineData(string apiDataMinerConfigFileLocation)
        {
            var databaseConnectionString = ConfigurationManager.AppSettings["SQLAzureDatabaseEntityFrameworkConnectionString"];
            var wattTimeApiKey           = ConfigurationManager.AppSettings["wattTimeApiKey"];
            var wundergroundApiKey       = ConfigurationManager.AppSettings["wundergroundApiKey"];
            var wattTimeApiV2Url         = ConfigurationManager.AppSettings["WattTimeApiV2Url"];
            var wattTimeUsername         = ConfigurationManager.AppSettings["WattTimeUsername"];
            var wattTimePassword         = ConfigurationManager.AppSettings["WattTimePassword"];
            var wattTimeEmail            = ConfigurationManager.AppSettings["WattTimeEmail"];
            var wattTimeOrganization     = ConfigurationManager.AppSettings["WattTimeOrganization"];
            var darkSkyApiUrl            = ConfigurationManager.AppSettings["DarkSkyApiUrl"];
            var darkSkyApiKey            = ConfigurationManager.AppSettings["DarkSkyApiKey"];
            var apiDataMiner             = new ApiDataMiner(databaseConnectionString);

            apiDataMiner.ParseMinerSettingsFileAndMineData(apiDataMinerConfigFileLocation, wattTimeApiKey, wundergroundApiKey, darkSkyApiKey, wattTimeUsername, wattTimePassword, wattTimeEmail,
                                                           wattTimeOrganization);
        }
        public void TestParseMinerSettingsFileAndMineData()
        {
            // Arrange
            var ConfigPath = @".\ApiDataMinerConfigs\ApiDataMinerConfigs.xml";

            // Act
            var apiDataMiner = new ApiDataMiner(databaseConnectionString);

            apiDataMiner.ParseMinerSettingsFileAndMineData(ConfigPath);

            // Assert
            using (var streamReader = new StreamReader(ConfigPath))
            {
                var xmlSerializer = new XmlSerializer(typeof(ApiMinerConfigLayout));
                var minerConfigs  = (ApiMinerConfigLayout)xmlSerializer.Deserialize(streamReader);

                foreach (var regionConfiguration in minerConfigs.Regions)
                {
                    // Verify emissions were mined successfully for each region in the Config File
                    if (regionConfiguration.EmissionsMiningRegion != null)
                    {
                        var emissionsRegionName       = regionConfiguration.EmissionsMiningRegion.friendlyName;
                        var timeZone                  = regionConfiguration.EmissionsMiningRegion.TimeZone;
                        var regionLat                 = regionConfiguration.EmissionsMiningRegion.Latitude;
                        var regionLong                = regionConfiguration.EmissionsMiningRegion.Longitude;
                        var regionWattTimeName        = regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation;
                        var wattTimeApiUrl            = regionConfiguration.EmissionsMiningRegion.ApiUrl;
                        var wattTimeApiKey            = regionConfiguration.EmissionsMiningRegion.ApiKey;
                        var selfThrottlingMethod      = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
                        var maxNumberOfCallsPerMinute =
                            regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;
                        var startDateTime = DateTime.UtcNow.AddDays(-2);
                        var endDateTime   = DateTime.UtcNow.AddDays(10);

                        var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute);
                        var results             = wattTimeInteraction.GetObservedMarginalCarbonResults(
                            wattTimeApiUrl,
                            regionWattTimeName,
                            startDateTime,
                            endDateTime,
                            null,
                            wattTimeApiKey);

                        int regionId;
                        using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
                        {
                            regionId =
                                _objectModel.AddEmissionsRegion(emissionsRegionName, timeZone, regionLat, regionLong)
                                .EmissionsRegionID;
                        }

                        using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
                        {
                            foreach (var result in results)
                            {
                                var dataPoint = _objectModel.FindCarbonEmissionsDataPoints(regionId, result.timestamp);
                                Assert.IsNotNull(dataPoint);
                            }
                        }
                    }

                    // Verify weather was mined successfully for each region in the Config File
                    if (regionConfiguration.WeatherMiningRegion != null)
                    {
                        var emissionsRegionName             = regionConfiguration.WeatherMiningRegion.friendlyName;
                        var timeZone                        = regionConfiguration.WeatherMiningRegion.TimeZone;
                        var regionLat                       = regionConfiguration.WeatherMiningRegion.Latitude;
                        var regionLong                      = regionConfiguration.WeatherMiningRegion.Longitude;
                        var weatherRegionWundergroundSubUrl =
                            regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl;
                        var wundergroundApiUrl        = regionConfiguration.WeatherMiningRegion.ApiUrl;
                        var wundergroundApiKey        = regionConfiguration.WeatherMiningRegion.ApiKey;
                        var selfThrottlingMethod      = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
                        var maxNumberOfCallsPerMinute =
                            regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;

                        var wundergroundWeatherInteraction = new WundergroundWeatherInteraction(
                            selfThrottlingMethod,
                            maxNumberOfCallsPerMinute);
                        var results =
                            wundergroundWeatherInteraction.GetTenDayHourlyForecastWeatherData(
                                wundergroundApiUrl,
                                weatherRegionWundergroundSubUrl,
                                wundergroundApiKey);

                        int regionId;
                        using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
                        {
                            regionId =
                                _objectModel.AddWeatherRegion(emissionsRegionName, timeZone, regionLat, regionLong)
                                .WeatherRegionID;
                        }

                        using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
                        {
                            foreach (var result in results)
                            {
                                var dataPoint = _objectModel.FindWeatherDataPoints(
                                    regionId,
                                    result.observationDateTime);
                                Assert.IsNotNull(dataPoint);
                            }
                        }
                    }
                }
            }
        }