public void TestMineHistoricWeatherValues() { // Arrange var regionSubUrl = "us/nj/princeton"; string smartGridRegionName = "PJM"; var timeZone = "Eastern Standard Time"; var regionLat = 40.348444276169; var regionLong = -74.6428556442261; var startDateTime = DateTime.Now.AddDays(-3); var endDateTime = DateTime.Now.AddDays(-1); var wundergroundApiUrl = CloudConfigurationManager.GetSetting("WundergroundApiUrl"); var wundergroundApiKey = CloudConfigurationManager.GetSetting("WundergroundApiKey"); var selfThrottlingMethod = "AzureTableStorageCallRecollection"; var maxNumberOfCallsPerMinute = 5; int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddWeatherRegion(smartGridRegionName, timeZone, regionLat, regionLong).WeatherRegionID; } var wundergroundWeatherInteraction = new WundergroundWeatherInteraction( selfThrottlingMethod, maxNumberOfCallsPerMinute); WeatherDataMiner weatherDataMiner = new WeatherDataMiner( wundergroundApiUrl, wundergroundApiKey, selfThrottlingMethod, databaseConnectionString, maxNumberOfCallsPerMinute, wundergroundWeatherInteraction); // Act weatherDataMiner.MineHistoricWeatherValues(startDateTime, endDateTime, regionSubUrl, regionId); // Assert // Verify that each data point has been recorded in the database var results = wundergroundWeatherInteraction.GetHistoricWeatherData( wundergroundApiUrl, regionSubUrl, wundergroundApiKey, startDateTime, endDateTime); using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { foreach (var result in results) { var dataPoint = _objectModel.FindWeatherDataPoints(regionId, result.observationDateTime); Assert.IsNotNull(dataPoint); } } }
public void TestMineHistoricWeatherValues_ByWundergroundLocationName() { // Arrange var regionSubUrl = "ie/dublin"; var latitude = 53.3498; var longtitude = -6.2603; var smartGridRegionName = "Ireland"; var timeZone = "GMT Standard Time"; var startDateTime = DateTime.Now.AddDays(-10); var endDateTime = DateTime.Now.AddDays(-1); var wundergroundApiUrl = CloudConfigurationManager.GetSetting("WundergroundApiUrl"); var wundergroundApiKey = CloudConfigurationManager.GetSetting("WundergroundApiKey"); var selfThrottlingMethod = "AzureTableStorageCallRecollection"; var maxNumberOfCallsPerMinute = 5; int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddWeatherRegion(smartGridRegionName, timeZone, regionLat, regionLong, regionSubUrl).WeatherRegionID; } var wundergroundWeatherInteraction = new WundergroundWeatherInteraction( selfThrottlingMethod, maxNumberOfCallsPerMinute); WundergroundWeatherDataMiner weatherDataMiner = new WundergroundWeatherDataMiner( wundergroundApiUrl, wundergroundApiKey, selfThrottlingMethod, databaseConnectionString, maxNumberOfCallsPerMinute, wundergroundWeatherInteraction); // Act weatherDataMiner.MineHistoricWeatherValues(startDateTime, endDateTime, regionSubUrl, regionId); // Assert // Verify that each data point has been recorded in the database var results = wundergroundWeatherInteraction.GetHistoricWeatherData( wundergroundApiUrl, regionSubUrl, wundergroundApiKey, startDateTime, endDateTime); using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { foreach (var result in results) { var dataPoint = _objectModel.FindWeatherDataPoint(regionId, result.observationDateTime); Assert.IsNotNull(dataPoint); } } }
public void TestMineHistoricWeatherValues_ByGPS() { // Arrange var startDateTime = new DateTime(2017, 1, 1); // DateTime.Now.AddDays(-10); var endDateTime = new DateTime(2017, 1, 2); // var startDateTime = DateTime.Now.AddDays(-1); var latitude = 58.279231; var longtitude = 6.892410; var smartGridRegionName = "Norway_Oye"; var timeZone = "Central European Standard Time"; var wundergroundApiUrl = CloudConfigurationManager.GetSetting("WundergroundApiUrl"); var wundergroundApiKey = CloudConfigurationManager.GetSetting("WundergroundApiKey"); var selfThrottlingMethod = "AzureTableStorageCallRecollection"; var maxNumberOfCallsPerMinute = 5; int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddWeatherRegion(smartGridRegionName, timeZone, latitude, longtitude, null).WeatherRegionID; } var wundergroundWeatherInteraction = new WundergroundWeatherInteraction( selfThrottlingMethod, maxNumberOfCallsPerMinute); WeatherDataMiner weatherDataMiner = new WeatherDataMiner( wundergroundApiUrl, wundergroundApiKey, selfThrottlingMethod, databaseConnectionString, maxNumberOfCallsPerMinute, wundergroundWeatherInteraction); // Act weatherDataMiner.MineHistoricWeatherValues(startDateTime, endDateTime, latitude, longtitude, regionId); // Assert }
/// <summary> /// Take a config element containing the data for a region to be mined, mine the data and save it in the database /// </summary> /// <param name="regionConfiguration"></param> public void MineRegionData(ApiMinerConfigLayoutRegion regionConfiguration) { var regionGroupingName = regionConfiguration.friendlyName; using (new TimedOperation( $"Beginning Mining of all data for Region {regionGroupingName}", "ApiDataMiner.MineRegionData()")) { // Mine the regions emissions if an emissions node was supplied int?emissionsRegionId = null; if (regionConfiguration.EmissionsMiningRegion != null) { var friendlyName = regionConfiguration.EmissionsMiningRegion.friendlyName; using ( new TimedOperation( $"Beginning Mining of emissions data for Region {friendlyName}", "ApiDataMiner.MineRegionData()")) { var timeZone = regionConfiguration.EmissionsMiningRegion.TimeZone; var regionLat = regionConfiguration.EmissionsMiningRegion.Latitude; var regionLong = regionConfiguration.EmissionsMiningRegion.Longitude; var regionWattTimeName = regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation; var wattTimeApiUrl = regionConfiguration.EmissionsMiningRegion.ApiUrl; var wattTimeApiKey = regionConfiguration.EmissionsMiningRegion.ApiKey; var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod; var maxNumberOfCallsPerMinute = regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute; var historicStartDateTime = DateTime.UtcNow.AddDays(-15); var historicEndDateTime = DateTime.UtcNow.AddDays(1); var forecastStartDateTime = DateTime.UtcNow.AddDays(-2); var forecastEndDateTime = DateTime.UtcNow.AddDays(10); using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { emissionsRegionId = _objectModel.AddEmissionsRegion(friendlyName, timeZone, regionLat, regionLong) .EmissionsRegionID; CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner( wattTimeApiUrl, wattTimeApiKey, selfThrottlingMethod, this.DatabaseConnectionString, maxNumberOfCallsPerMinute); // Mine Recent Actual Data carbonEmissionsMiner.MineHistoricCarbonResults( historicStartDateTime, historicEndDateTime, regionWattTimeName, (int)emissionsRegionId); // Mine Forecast Data carbonEmissionsMiner.MineForecastMarginalCarbonResults( forecastStartDateTime, forecastEndDateTime, regionWattTimeName, (int)emissionsRegionId); } } } // Mine the regions emissions if an emissions node was supplied int?weatherRegionId = null; if (regionConfiguration.WeatherMiningRegion != null) { var friendlyName = regionConfiguration.WeatherMiningRegion.friendlyName; using ( new TimedOperation( $"Beginning Mining of weather data for Region {friendlyName}", "ApiDataMiner.MineRegionData()")) { var timeZone = regionConfiguration.WeatherMiningRegion.TimeZone; var regionLat = regionConfiguration.WeatherMiningRegion.Latitude; var regionLong = regionConfiguration.WeatherMiningRegion.Longitude; var weatherRegionWundergroundSubUrl = regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl; var wundergroundApiUrl = regionConfiguration.WeatherMiningRegion.ApiUrl; var wundergroundApiKey = regionConfiguration.WeatherMiningRegion.ApiKey; var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod; var maxNumberOfCallsPerMinute = regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute; var historicStartDateTime = DateTime.UtcNow.AddDays(-1); var historicEndDateTime = DateTime.UtcNow.AddDays(1); using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { weatherRegionId = _objectModel.AddWeatherRegion( friendlyName, timeZone, regionLat, regionLong, weatherRegionWundergroundSubUrl).WeatherRegionID; WeatherDataMiner weatherDataMiner = new WeatherDataMiner( wundergroundApiUrl, wundergroundApiKey, selfThrottlingMethod, this.DatabaseConnectionString, maxNumberOfCallsPerMinute); // Mine Recent Actual Data weatherDataMiner.MineHistoricWeatherValues( historicStartDateTime, historicEndDateTime, weatherRegionWundergroundSubUrl, (int)weatherRegionId); // Mine Forecast Data weatherDataMiner.MineTenDayHourlyForecastWeatherValues( weatherRegionWundergroundSubUrl, (int)weatherRegionId); } } } // Group the Emissions and Weather regions if they haven't already been grouped using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { _objectModel.AddMarketWeatherEmissionsRegionMapping( regionGroupingName, null, weatherRegionId, emissionsRegionId); } } }
public void TestParseMinerSettingsFileAndMineData() { // Arrange var ConfigPath = @".\ApiDataMinerConfigs\ApiDataMinerConfigs.xml"; // Act var apiDataMiner = new ApiDataMiner(databaseConnectionString); apiDataMiner.ParseMinerSettingsFileAndMineData(ConfigPath); // Assert using (var streamReader = new StreamReader(ConfigPath)) { var xmlSerializer = new XmlSerializer(typeof(ApiMinerConfigLayout)); var minerConfigs = (ApiMinerConfigLayout)xmlSerializer.Deserialize(streamReader); foreach (var regionConfiguration in minerConfigs.Regions) { // Verify emissions were mined successfully for each region in the Config File if (regionConfiguration.EmissionsMiningRegion != null) { var emissionsRegionName = regionConfiguration.EmissionsMiningRegion.friendlyName; var timeZone = regionConfiguration.EmissionsMiningRegion.TimeZone; var regionLat = regionConfiguration.EmissionsMiningRegion.Latitude; var regionLong = regionConfiguration.EmissionsMiningRegion.Longitude; var regionWattTimeName = regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation; var wattTimeApiUrl = regionConfiguration.EmissionsMiningRegion.ApiUrl; var wattTimeApiKey = regionConfiguration.EmissionsMiningRegion.ApiKey; var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod; var maxNumberOfCallsPerMinute = regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute; var startDateTime = DateTime.UtcNow.AddDays(-2); var endDateTime = DateTime.UtcNow.AddDays(10); var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute); var results = wattTimeInteraction.GetObservedMarginalCarbonResults( wattTimeApiUrl, regionWattTimeName, startDateTime, endDateTime, null, wattTimeApiKey); int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddEmissionsRegion(emissionsRegionName, timeZone, regionLat, regionLong) .EmissionsRegionID; } using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { foreach (var result in results) { var dataPoint = _objectModel.FindCarbonEmissionsDataPoints(regionId, result.timestamp); Assert.IsNotNull(dataPoint); } } } // Verify weather was mined successfully for each region in the Config File if (regionConfiguration.WeatherMiningRegion != null) { var emissionsRegionName = regionConfiguration.WeatherMiningRegion.friendlyName; var timeZone = regionConfiguration.WeatherMiningRegion.TimeZone; var regionLat = regionConfiguration.WeatherMiningRegion.Latitude; var regionLong = regionConfiguration.WeatherMiningRegion.Longitude; var weatherRegionWundergroundSubUrl = regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl; var wundergroundApiUrl = regionConfiguration.WeatherMiningRegion.ApiUrl; var wundergroundApiKey = regionConfiguration.WeatherMiningRegion.ApiKey; var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod; var maxNumberOfCallsPerMinute = regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute; var wundergroundWeatherInteraction = new WundergroundWeatherInteraction( selfThrottlingMethod, maxNumberOfCallsPerMinute); var results = wundergroundWeatherInteraction.GetTenDayHourlyForecastWeatherData( wundergroundApiUrl, weatherRegionWundergroundSubUrl, wundergroundApiKey); int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddWeatherRegion(emissionsRegionName, timeZone, regionLat, regionLong) .WeatherRegionID; } using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { foreach (var result in results) { var dataPoint = _objectModel.FindWeatherDataPoints( regionId, result.observationDateTime); Assert.IsNotNull(dataPoint); } } } } } }
public void TestMineForecastcWeatherValues_ByGPS() { // Arrange var startDateTime = DateTime.UtcNow; var endDateTime = DateTime.UtcNow.AddDays(10); var latitude = 53.3498; var longtitude = -6.2603; var smartGridRegionName = "Ireland"; var timeZone = "GMT Standard Time"; var darkSkyApiUrl = CloudConfigurationManager.GetSetting("DarkSkyApiUrl"); var darkSkyApiKey = CloudConfigurationManager.GetSetting("DarkSkyApiKey"); var selfThrottlingMethod = "AzureTableStorageCallRecollection"; var maxNumberOfCallsPerMinute = 5; var maxNumberOfCallsPerDay = 500; int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddWeatherRegion(smartGridRegionName, timeZone, latitude, longtitude, null).WeatherRegionID; } var darkSkyWeatherInteraction = new DarkSkyWeatherInteraction( selfThrottlingMethod, maxNumberOfCallsPerMinute, maxNumberOfCallsPerDay); DarkSkyWeatherDataMiner weatherDataMiner = new DarkSkyWeatherDataMiner( darkSkyApiUrl, darkSkyApiKey, selfThrottlingMethod, databaseConnectionString, maxNumberOfCallsPerMinute, maxNumberOfCallsPerDay, darkSkyWeatherInteraction); // Act weatherDataMiner.MineForecastWeatherValues(startDateTime, endDateTime, latitude, longtitude, regionId); // Assert //Verify that each data point has been recorded in the database var results = darkSkyWeatherInteraction.GetForecastWeatherData( darkSkyApiUrl, darkSkyApiKey, latitude, longtitude, startDateTime, endDateTime); using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { foreach (var result in results) { var dataPoint = _objectModel.FindWeatherDataPoint(regionId, result.dateTime); Assert.IsNotNull(dataPoint); } } }
/// <summary> /// Take a config element containing the data for a region to be mined, mine the data and save it in the database /// </summary> /// <param name="regionConfiguration"></param> /// <param name="wattTimeApiKeyOverride">Optional wattTimeApiKey to Override what's in the MinerConfigXML File</param> /// <param name="wundergroundApiKeyOverride">Optional WundergroundApiKey to Override what's in the MinerConfigXML File</param> public void MineRegionData(ApiMinerConfigLayoutRegion regionConfiguration, string wattTimeApiKeyOverride = null, string wundergroundApiKeyOverride = null) { var regionGroupingName = regionConfiguration.friendlyName; using (new TimedOperation( $"Beginning Mining of all data for Region {regionGroupingName}", "ApiDataMiner.MineRegionData()")) { // Mine the regions emissions if an emissions node was supplied int?emissionsRegionId = null; if (regionConfiguration.EmissionsMiningRegion != null) { var friendlyName = regionConfiguration.EmissionsMiningRegion.friendlyName; using ( new TimedOperation( $"Beginning Mining of emissions data for Region {friendlyName}", "ApiDataMiner.MineRegionData()")) { var timeZone = regionConfiguration.EmissionsMiningRegion.TimeZone; var regionLat = regionConfiguration.EmissionsMiningRegion.Latitude; var regionLong = regionConfiguration.EmissionsMiningRegion.Longitude; var regionWattTimeName = regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation; var wattTimeApiUrl = regionConfiguration.EmissionsMiningRegion.ApiUrl; string wattTimeApiKey = null; if (string.IsNullOrEmpty(wattTimeApiKeyOverride) || wattTimeApiKeyOverride.Equals("none")) { wattTimeApiKey = regionConfiguration.EmissionsMiningRegion.ApiKey; } else { wattTimeApiKey = wattTimeApiKeyOverride; } var selfThrottlingMethod = regionConfiguration.EmissionsMiningRegion.SelfThrottlingMethod; var maxNumberOfCallsPerMinute = regionConfiguration.EmissionsMiningRegion.MaxNumberOfCallsPerMinute; var historicStartDateTime = DateTime.UtcNow.AddDays(-15); var historicEndDateTime = DateTime.UtcNow.AddDays(1); var forecastStartDateTime = DateTime.UtcNow.AddDays(-2); var forecastEndDateTime = DateTime.UtcNow.AddDays(10); if (!string.IsNullOrEmpty(wattTimeApiKey) && !wattTimeApiKey.Equals("none")) { Logger.Information( $"About to add Emissions Region and Mine Carbon Emissions Data for {regionWattTimeName} from WattTime URL {wattTimeApiUrl} from {historicStartDateTime} to {historicEndDateTime} for historic data and insert them into the database", "ApiDataMiner.MineRegionData()"); using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { emissionsRegionId = _objectModel.AddEmissionsRegion(friendlyName, timeZone, regionLat, regionLong, regionWattTimeName) .EmissionsRegionID; CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner( wattTimeApiUrl, wattTimeApiKey, selfThrottlingMethod, this.DatabaseConnectionString, maxNumberOfCallsPerMinute); // Mine Recent Actual Data carbonEmissionsMiner.MineHistoricCarbonResults( historicStartDateTime, historicEndDateTime, regionWattTimeName, (int)emissionsRegionId); // Mine Forecast Data carbonEmissionsMiner.MineForecastMarginalCarbonResults( forecastStartDateTime, forecastEndDateTime, regionWattTimeName, (int)emissionsRegionId); } } else { Logger.Information( $"No WattTime Api Key was specified. Skipping this region for Emissions.", "RunAsync()"); } } } // Mine the regions weather if a weather node was supplied int?weatherRegionId = null; if (regionConfiguration.WeatherMiningRegion != null) { var friendlyName = regionConfiguration.WeatherMiningRegion.friendlyName; using ( new TimedOperation( $"Beginning Mining of weather data for Region {friendlyName}", "ApiDataMiner.MineRegionData()")) { var timeZone = regionConfiguration.WeatherMiningRegion.TimeZone; var regionLat = regionConfiguration.WeatherMiningRegion.Latitude; var regionLong = regionConfiguration.WeatherMiningRegion.Longitude; var weatherRegionWundergroundSubUrl = regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl; var wundergroundApiUrl = regionConfiguration.WeatherMiningRegion.ApiUrl; string wundergroundApiKey = null; if (string.IsNullOrEmpty(wundergroundApiKeyOverride) || wundergroundApiKeyOverride.Equals("none")) { wundergroundApiKey = regionConfiguration.WeatherMiningRegion.ApiKey; } else { wundergroundApiKey = wundergroundApiKeyOverride; } var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod; var maxNumberOfCallsPerMinute = regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute; var historicStartDateTime = DateTime.UtcNow.AddDays(-1); var historicEndDateTime = DateTime.UtcNow.AddDays(1); if (!string.IsNullOrEmpty(wundergroundApiKey) && !wundergroundApiKey.Equals("none")) { Logger.Information( $"About to add Emissions Region and Mine Carbon Emissions Data for {friendlyName} from Wunderground URL {weatherRegionWundergroundSubUrl} from {historicStartDateTime} to {historicEndDateTime} for historic data and insert them into the database", "ApiDataMiner.MineRegionData()"); using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { weatherRegionId = _objectModel.AddWeatherRegion( friendlyName, timeZone, regionLat, regionLong, weatherRegionWundergroundSubUrl).WeatherRegionID; WeatherDataMiner weatherDataMiner = new WeatherDataMiner( wundergroundApiUrl, wundergroundApiKey, selfThrottlingMethod, this.DatabaseConnectionString, maxNumberOfCallsPerMinute); switch (regionConfiguration.WeatherMiningRegion.MiningMethod) { case "GPS": // Mine Recent Actual Data weatherDataMiner.MineHistoricWeatherValues( historicStartDateTime, historicEndDateTime, regionLat, regionLong, (int)weatherRegionId); // Mine Forecast Data weatherDataMiner.MineTenDayHourlyForecastWeatherValues( regionLat, regionLong, (int)weatherRegionId); break; case "WundergroundPageSubUrl": default: // Mine Recent Actual Data weatherDataMiner.MineHistoricWeatherValues( historicStartDateTime, historicEndDateTime, weatherRegionWundergroundSubUrl, (int)weatherRegionId); // Mine Forecast Data weatherDataMiner.MineTenDayHourlyForecastWeatherValues( weatherRegionWundergroundSubUrl, (int)weatherRegionId); break; } } } else { Logger.Information( $"No Wunderground Api Key was specified. Skipping this region for Weather.", "RunAsync()"); } } } // Group the Emissions and Weather regions if they haven't already been grouped using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { _objectModel.AddMarketWeatherEmissionsRegionMapping( regionGroupingName, null, weatherRegionId, emissionsRegionId); } } }