public void TestMineHistoricWeatherValues() { // Arrange var regionSubUrl = "us/nj/princeton"; string smartGridRegionName = "PJM"; var timeZone = "Eastern Standard Time"; var regionLat = 40.348444276169; var regionLong = -74.6428556442261; var startDateTime = DateTime.Now.AddDays(-3); var endDateTime = DateTime.Now.AddDays(-1); var wundergroundApiUrl = CloudConfigurationManager.GetSetting("WundergroundApiUrl"); var wundergroundApiKey = CloudConfigurationManager.GetSetting("WundergroundApiKey"); var selfThrottlingMethod = "AzureTableStorageCallRecollection"; var maxNumberOfCallsPerMinute = 5; int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddWeatherRegion(smartGridRegionName, timeZone, regionLat, regionLong).WeatherRegionID; } var wundergroundWeatherInteraction = new WundergroundWeatherInteraction( selfThrottlingMethod, maxNumberOfCallsPerMinute); WeatherDataMiner weatherDataMiner = new WeatherDataMiner( wundergroundApiUrl, wundergroundApiKey, selfThrottlingMethod, databaseConnectionString, maxNumberOfCallsPerMinute, wundergroundWeatherInteraction); // Act weatherDataMiner.MineHistoricWeatherValues(startDateTime, endDateTime, regionSubUrl, regionId); // Assert // Verify that each data point has been recorded in the database var results = wundergroundWeatherInteraction.GetHistoricWeatherData( wundergroundApiUrl, regionSubUrl, wundergroundApiKey, startDateTime, endDateTime); using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { foreach (var result in results) { var dataPoint = _objectModel.FindWeatherDataPoints(regionId, result.observationDateTime); Assert.IsNotNull(dataPoint); } } }
public IHttpActionResult Get([FromUri] string id, [FromUri] DateTime?startDateTime = null, [FromUri] DateTime?endDateTime = null, [FromUri] double dateTimeFlexabilityInMinutes = 0) { try { // Set default start and end datetimes string emissionsRegionFriendlyName = id.IsNullOrWhiteSpace() ? "US_PJM" : id; var startDateTimeProcessed = startDateTime ?? DateTime.UtcNow.AddHours(-3); var endDateTimeProcessed = endDateTime ?? DateTime.UtcNow.AddHours(1); // Query database var databaseConnectionString = CloudConfigurationManager.GetSetting("SQLAzureDatabaseEntityFrameworkConnectionString"); using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { var regionId = _objectModel.FindEmissionsRegion(emissionsRegionFriendlyName).EmissionsRegionID; var result = _objectModel.FindCarbonEmissionsRelativeMeritDataPoints(regionId, startDateTimeProcessed, endDateTimeProcessed, dateTimeFlexabilityInMinutes).ToList(); // Convert the final list to objects of the common data type in the SmartEnergyAzureDataTypes NuGet package var webResult = this.ConvertToCarbonEmissionsRelativeMeritWebDataPoints(result); if (result == null) { return(this.NotFound()); } return(Ok(webResult)); } } catch (Exception e) { throw new Exception("Sorry - an exception occured executing the request"); } }
public void TestMineHistoricWeatherValues_ByWundergroundLocationName() { // Arrange var regionSubUrl = "ie/dublin"; var latitude = 53.3498; var longtitude = -6.2603; var smartGridRegionName = "Ireland"; var timeZone = "GMT Standard Time"; var startDateTime = DateTime.Now.AddDays(-10); var endDateTime = DateTime.Now.AddDays(-1); var wundergroundApiUrl = CloudConfigurationManager.GetSetting("WundergroundApiUrl"); var wundergroundApiKey = CloudConfigurationManager.GetSetting("WundergroundApiKey"); var selfThrottlingMethod = "AzureTableStorageCallRecollection"; var maxNumberOfCallsPerMinute = 5; int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddWeatherRegion(smartGridRegionName, timeZone, regionLat, regionLong, regionSubUrl).WeatherRegionID; } var wundergroundWeatherInteraction = new WundergroundWeatherInteraction( selfThrottlingMethod, maxNumberOfCallsPerMinute); WundergroundWeatherDataMiner weatherDataMiner = new WundergroundWeatherDataMiner( wundergroundApiUrl, wundergroundApiKey, selfThrottlingMethod, databaseConnectionString, maxNumberOfCallsPerMinute, wundergroundWeatherInteraction); // Act weatherDataMiner.MineHistoricWeatherValues(startDateTime, endDateTime, regionSubUrl, regionId); // Assert // Verify that each data point has been recorded in the database var results = wundergroundWeatherInteraction.GetHistoricWeatherData( wundergroundApiUrl, regionSubUrl, wundergroundApiKey, startDateTime, endDateTime); using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { foreach (var result in results) { var dataPoint = _objectModel.FindWeatherDataPoint(regionId, result.observationDateTime); Assert.IsNotNull(dataPoint); } } }
/// <summary> /// Retrieve historic marginal emissions results between the given dates and store them in the database /// </summary> /// <param name="startDateTime">Optional StartDateTime. If not supplied, a default value will be used</param> /// <param name="endDateTime">Optional endDateTime. If not supplied, a default value will be used</param> /// <param name="regionWattTimeName">Abbreviation for the required region (e.g. "PJM"). See https://api.watttime.org/faq/#where </param> /// <param name="regionId">regionId of this region in the application's database</param> public void MineHistoricMarginalCarbonResults( DateTime?startDateTime, DateTime?endDateTime, string regionWattTimeName, int regionId) { var historicStartDateTime = startDateTime ?? DateTime.UtcNow.AddDays(-2); var historicEndDateTime = endDateTime ?? DateTime.UtcNow.AddMinutes(15); try { var results = this.wattTimeEmissionsInteraction.GetObservedMarginalCarbonResults( wattTimeApiUrl, regionWattTimeName, historicStartDateTime, historicEndDateTime, null, wattTimeApiKey); Logger.Information( $"Received {results.Count} HistoricMarginalCarbonResults Results for {regionWattTimeName} from WattTime. Inserting them into the database", "CarbonEmissionsMiner.MineHistoricMarginalCarbonResults()"); // Insert results in the database using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { foreach (var res in results) { var dateTime = res.timestamp; var marginalCarbon = res.marginal_carbon.value; var units = res.marginal_carbon.units; if (marginalCarbon != null) { var marginalCarbonMetric = units == "lb/MW" ? this.wattTimeEmissionsInteraction .ConvertLbsPerMWhTo_GCo2PerkWh((double)marginalCarbon) : marginalCarbon; _objectModel.InsertOrUpdateCarbonEmissionsDataPoints( regionId, dateTime, null, false, marginalCarbonMetric, false); } } } } catch (Exception e) { Logger.Error( $"CarbonEmissionsMiner: MineHistoricMarginalCarbonResults(): Exception encountered while emissions Data figures for {regionWattTimeName} between {historicStartDateTime} and {historicEndDateTime}.", "CarbonEmissionsMiner.MineHistoricMarginalCarbonResults()", null, e); } }
private void InsertWeatherValuesIntoDatabase(int regionId, List <HourlyDatum> results, bool areForecastRows = false) { using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { foreach (var res in results) { var dateTime = res.dateTime; _objectModel.InsertOrUpdateWeatherDataPoints( regionId, dateTime, res.temperature, res.dewPoint, res.windSpeed, null, res.windBearing, res.apparentTemperature, res.visibility, null, null, null, res.pressure, res.humidity, res.summary, areForecastRows); } } }
/// <summary> /// Retrieve forecast marginal emissions results between the given dates and store them in the database /// </summary> /// <param name="startDateTime">Optional StartDateTime. If not supplied, a default value will be used</param> /// <param name="endDateTime">Optional endDateTime. If not supplied, a default value will be used</param> /// <param name="regionWattTimeName">Abbreviation for the required region (e.g. "PJM"). See https://api.watttime.org/faq/#where </param> /// <param name="regionId">regionId of this region in the application's database</param> public void MineForecastMarginalCarbonResults( DateTime?startDateTime, DateTime?endDateTime, string regionWattTimeName, int regionId) { var historicStartDateTime = startDateTime ?? DateTime.UtcNow.AddDays(-2); var historicEndDateTime = endDateTime ?? DateTime.UtcNow.AddMinutes(15); try { Logger.Information( $"Mining Forecast Marginal Carbon Results for {regionWattTimeName} from WattTime URL {this.wattTimeApiUrl}.", "CarbonEmissionsMiner.MineForecastMarginalCarbonResults()"); var results = this.wattTimeEmissionsInteraction.GetForecastMarginalCarbonResults( WattTimeV2ApiUrl, regionWattTimeName, WattTimeUsername, WattTimePassword, historicStartDateTime, historicEndDateTime, null); Logger.Information( $"Received {results.Count} ForecastMarginalCarbonResults Results for {regionWattTimeName} from WattTime. Inserting them into the database", "CarbonEmissionsMiner.ForecastMarginalCarbonResults()"); // Insert results in the database using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { foreach (var res in results) { var dateTime = res.point_time; var marginalCarbon = res.value; var marginalCarbonMetric = this.wattTimeEmissionsInteraction .ConvertLbsPerMWhTo_GCo2PerkWh((double)marginalCarbon); _objectModel.InsertOrUpdateCarbonEmissionsDataPoints( regionId, dateTime, null, null, null, marginalCarbonMetric); } } } catch (Exception e) { Logger.Error( $"CarbonEmissionsMiner: MineForecastMarginalCarbonResults(): Exception encountered while emissions Data figures for {regionWattTimeName} between {historicStartDateTime} and {historicEndDateTime}.", "CarbonEmissionsMiner.MineForecastMarginalCarbonResults()", null, e); } }
/// <summary> /// Mine Or Calculate Relative Merit Data for emissions values between the given dates and store them in the database. /// The Relative Merit is a value between 0 (meaning best) and 1 (being worst). /// </summary> /// <param name="startDateTime">Optional StartDateTime. If not supplied, a default value will be used</param> /// <param name="endDateTime">Optional endDateTime. If not supplied, a default value will be used</param> /// <param name="regionWattTimeName">Abbreviation for the required region (e.g. "PJM"). See https://api.watttime.org/faq/#where </param> /// <param name="regionId">regionId of this region in the application's database</param> public void MineOrCalculateHistoricRelativeMeritData( double latitude, double longitude, DateTime?startDateTime, DateTime?endDateTime, string regionWattTimeName, int regionId) { var historicStartDateTime = startDateTime ?? DateTime.UtcNow.AddDays(-2); var historicEndDateTime = endDateTime ?? DateTime.UtcNow.AddMinutes(15); try { Logger.Information( $"Entering method for {regionWattTimeName} from WattTime URL {this.wattTimeApiUrl} with method of relative data retrieval / calculation of {this.RelativeMeritDataSource}.", "CarbonEmissionsMiner.MineOrCalculateHistoricRelativeMeritData()"); switch (this.RelativeMeritDataSource) { case "WattTime": throw new NotImplementedException("WattTime does not currently offer historic relative merit data querying"); break; case "CustomInternalCalculation": var calculatedHistoricRelativeMeritResults = this.CalculateHistoricRelativeMeritDataResults( regionId, historicStartDateTime, historicEndDateTime); // Insert results in the database using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { foreach (var res in calculatedHistoricRelativeMeritResults) { _objectModel.InsertOrUpdateCarbonEmissionsRelativeMeritDataPoints( regionId, res.Timestamp, res.EmissionsRelativeMerit, res.EmissionsRelativeMerit_Forcast); } } break; default: Logger.Information( $"No known defined method of MineOrCalculateHistoricRelativeMeritData supplied to method. Not mining or calculating Historic Relative Merit Data for this region ({regionWattTimeName})", "CarbonEmissionsMiner.MineOrCalculateHistoricRelativeMeritData()"); return; } } catch (Exception e) { Logger.Error( $"CarbonEmissionsMiner: MineForecastMarginalCarbonResults(): Exception encountered while emissions Data figures for {regionWattTimeName} between {historicStartDateTime} and {historicEndDateTime}.", "CarbonEmissionsMiner.MineForecastMarginalCarbonResults()", null, e); } }
/// <summary> /// InsertTenDatForecastValuesIntoDatabase /// </summary> /// <param name="regionId"></param> /// <param name="results"></param> private void InsertTenDatForecastValuesIntoDatabase(int regionId, List <HourlyForecast> results) { using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { foreach (var res in results) { var dateTime = res.observationDateTime; double tempcelcuis; double.TryParse(res.temp.metric, out tempcelcuis); double dewPoint; double.TryParse(res.dewpoint.metric, out dewPoint); double windSpeedKmPh; double.TryParse(res.wspd.metric, out windSpeedKmPh); double windDirectionDegrees; double.TryParse(res.wdir.degrees, out windDirectionDegrees); double windChill; double.TryParse(res.windchill.metric, out windChill); double uvIndex; double.TryParse(res.uvi, out uvIndex); double snow; double.TryParse(res.snow.metric, out snow); double pressure; double.TryParse(res.mslp.metric, out pressure); double humidityPercentage; double.TryParse(res.humidity, out humidityPercentage); double precipitation; double.TryParse(res.qpf.metric, out precipitation); var conditionDescription = res.condition ?? string.Empty; // Clean up "-9999" values that sometimes come back if (windSpeedKmPh < 0) { windSpeedKmPh = -1; } _objectModel.InsertOrUpdateWeatherDataPoints( regionId, dateTime, tempcelcuis, dewPoint, windSpeedKmPh, null, // No wind gusts in Wunderground forecasts windDirectionDegrees, windChill, null, // No visibilty in Wunderground forecasts uvIndex, precipitation, snow, pressure, humidityPercentage, conditionDescription, true); } } }
public IEnumerable <string> Get() { try { var databaseConnectionString = CloudConfigurationManager.GetSetting("SQLAzureDatabaseEntityFrameworkConnectionString"); using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { var result = _objectModel.FindAllEmissionsRegions(); return(result); } } catch (Exception e) { throw new Exception("Sorry - an exception occured executing the request"); } }
public IHttpActionResult Get([FromUri] string id, [FromUri] DateTime?startDateTime = null, [FromUri] DateTime?endDateTime = null, [FromUri] bool returnOnlyNonNullSystemWideEmissionsDataPoints = false, [FromUri] bool returnOnlyNonNullMarginalEmissionsDataPoints = false) { try { // Set default start and end datetimes string emissionsRegionFriendlyName = id.IsNullOrWhiteSpace() ? "US_PJM" : id; var startDateTimeProcessed = startDateTime ?? DateTime.UtcNow.AddHours(-3); var endDateTimeProcessed = endDateTime ?? DateTime.UtcNow.AddHours(1); // Query database var databaseConnectionString = CloudConfigurationManager.GetSetting("SQLAzureDatabaseEntityFrameworkConnectionString"); using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { var regionId = _objectModel.FindEmissionsRegion(emissionsRegionFriendlyName).EmissionsRegionID; var result = _objectModel.FindCarbonEmissionsDataPoints(regionId, startDateTimeProcessed, endDateTimeProcessed).ToList(); // Remove any relavent results as per search parameters passed in - System Wide and Marginal Emissions if (returnOnlyNonNullSystemWideEmissionsDataPoints) { result.RemoveAll(x => x.SystemWideCO2Intensity_gCO2kWh == null); } if (returnOnlyNonNullMarginalEmissionsDataPoints) { result.RemoveAll(x => x.MarginalCO2Intensity_gCO2kWh == null); } // Convert the final list to objects of the common data type in the SmartEnergyAzureDataTypes NuGet package var webResult = this.ConvertToCarbonEmissionsWebDataPoints(result); if (result == null) { return(this.NotFound()); } return(Ok(webResult)); } } catch (Exception e) { throw new Exception("Sorry - an exception occured executing the request"); } }
public void TestMineHistoricWeatherValues_ByGPS() { // Arrange var startDateTime = new DateTime(2017, 1, 1); // DateTime.Now.AddDays(-10); var endDateTime = new DateTime(2017, 1, 2); // var startDateTime = DateTime.Now.AddDays(-1); var latitude = 58.279231; var longtitude = 6.892410; var smartGridRegionName = "Norway_Oye"; var timeZone = "Central European Standard Time"; var wundergroundApiUrl = CloudConfigurationManager.GetSetting("WundergroundApiUrl"); var wundergroundApiKey = CloudConfigurationManager.GetSetting("WundergroundApiKey"); var selfThrottlingMethod = "AzureTableStorageCallRecollection"; var maxNumberOfCallsPerMinute = 5; int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddWeatherRegion(smartGridRegionName, timeZone, latitude, longtitude, null).WeatherRegionID; } var wundergroundWeatherInteraction = new WundergroundWeatherInteraction( selfThrottlingMethod, maxNumberOfCallsPerMinute); WeatherDataMiner weatherDataMiner = new WeatherDataMiner( wundergroundApiUrl, wundergroundApiKey, selfThrottlingMethod, databaseConnectionString, maxNumberOfCallsPerMinute, wundergroundWeatherInteraction); // Act weatherDataMiner.MineHistoricWeatherValues(startDateTime, endDateTime, latitude, longtitude, regionId); // Assert }
/// <summary> /// Take a CSV File containing CarbonEmissionsDataPoint values, and add them as CarbonEmissionsDataPoint to the database /// </summary> /// <param name="folderContainingCsv"></param> /// <param name="csvFileName"></param> /// <param name="regionId">RegionId to which the CarbonEmissionsDataPoint values pertain</param> /// <param name="dateTimeFormat">DateTime format to be passed to DateTime.ParseExact(). E.g. "MM/dd/yyyy HH:mm:ss"</param> /// <param name="ignoreParsingErrors">True to ignore errors parsing individual rows. False to rethrow the exceptions.</param> /// <returns></returns> public void ImportCarbonResultsToDatabaseFromCsv( string folderContainingCsv, string csvFileName, int regionId, string dateTimeFormat = "MM/dd/yyyy HH:mm:ss", bool ignoreParsingErrors = true) { try { Logger.Information( $"Importing Historic Carbon Results for RegionId {regionId} from CSV {csvFileName}.", "CarbonEmissionsMiner.ImportCarbonResultsToDatabaseFromCsv()"); var results = ImportCarbonResultsFromCsv(folderContainingCsv, csvFileName, regionId, dateTimeFormat, ignoreParsingErrors); using (var objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { foreach (var result in results) { // Insert the value into the database objectModel.InsertOrUpdateCarbonEmissionsDataPoints( regionId, result.DateTimeUTC, result.SystemWideCO2Intensity_gCO2kWh, null, result.MarginalCO2Intensity_gCO2kWh, null); } } } catch (Exception e) { Logger.Error( $"CarbonEmissionsMiner: ImportCarbonResultsToDatabaseFromCsv(): Exception encountered importing emissions Data from CSV with name {csvFileName} in folder {folderContainingCsv} and inserting them to the database.", "CarbonEmissionsMiner.ImportCarbonResultsToDatabaseFromCsv()", null, e); } }
public void TestMineForecastcWeatherValues_ByGPS() { // Arrange var startDateTime = DateTime.UtcNow; var endDateTime = DateTime.UtcNow.AddDays(10); var latitude = 53.3498; var longtitude = -6.2603; var smartGridRegionName = "Ireland"; var timeZone = "GMT Standard Time"; var darkSkyApiUrl = CloudConfigurationManager.GetSetting("DarkSkyApiUrl"); var darkSkyApiKey = CloudConfigurationManager.GetSetting("DarkSkyApiKey"); var selfThrottlingMethod = "AzureTableStorageCallRecollection"; var maxNumberOfCallsPerMinute = 5; var maxNumberOfCallsPerDay = 500; int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddWeatherRegion(smartGridRegionName, timeZone, latitude, longtitude, null).WeatherRegionID; } var darkSkyWeatherInteraction = new DarkSkyWeatherInteraction( selfThrottlingMethod, maxNumberOfCallsPerMinute, maxNumberOfCallsPerDay); DarkSkyWeatherDataMiner weatherDataMiner = new DarkSkyWeatherDataMiner( darkSkyApiUrl, darkSkyApiKey, selfThrottlingMethod, databaseConnectionString, maxNumberOfCallsPerMinute, maxNumberOfCallsPerDay, darkSkyWeatherInteraction); // Act weatherDataMiner.MineForecastWeatherValues(startDateTime, endDateTime, latitude, longtitude, regionId); // Assert //Verify that each data point has been recorded in the database var results = darkSkyWeatherInteraction.GetForecastWeatherData( darkSkyApiUrl, darkSkyApiKey, latitude, longtitude, startDateTime, endDateTime); using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { foreach (var result in results) { var dataPoint = _objectModel.FindWeatherDataPoint(regionId, result.dateTime); Assert.IsNotNull(dataPoint); } } }
/// <summary> /// Mine historic weather records from the Wunderground Weather service for the given regionSubUrl (e.g. CA/San_Francisco) and add it to the database /// </summary> /// <param name="startDateTime">startDateTime</param> /// <param name="endDateTime">endDateTime</param> /// <param name="regionSubUrl">Sub Url of the region on the Wunderground weather API e.g. CA/San_Francisco</param> /// <param name="regionId">regionId of this region in the application's database</param> public void MineHistoricWeatherValues( DateTime?startDateTime, DateTime?endDateTime, string regionSubUrl, int regionId) { var historicStartDateTime = startDateTime ?? DateTime.Now.AddDays(-2); var historicEndDateTime = endDateTime ?? DateTime.Now.AddMinutes(15); try { var results = this.wundergroundWeatherInteraction.GetHistoricWeatherData( this.wundergroundApiUrl, regionSubUrl, this.wundergroundApiKey, historicStartDateTime, historicEndDateTime); Logger.Information( $"Received {results.Count} HistoricWeatherValues Results for region with SubUrl {regionSubUrl} from Wunderground. Inserting them into the database", "WeatherDataMiner.MineHistoricWeatherValues()"); // Insert results in the database using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { foreach (var res in results) { var dateTime = res.observationDateTime; double tempcelcuis; double.TryParse(res.tempm, out tempcelcuis); double dewPoint; double.TryParse(res.dewptm, out dewPoint); double humidityPercentage; double.TryParse(res.hum, out humidityPercentage); double windSpeedKmPh; double.TryParse(res.wspdm, out windSpeedKmPh); double windGustKmPh; double.TryParse(res.wgustm, out windGustKmPh); double windDirectionDegrees; double.TryParse(res.wdird, out windDirectionDegrees); double windChill; double.TryParse(res.windchillm, out windChill); double visibilityMetric; double.TryParse(res.vism, out visibilityMetric); double snow; double.TryParse(res.snow, out snow); double pressure; double.TryParse(res.pressurem, out pressure); double precipitation; double.TryParse(res.precipm, out precipitation); var conditionDescription = res.conds; _objectModel.InsertOrUpdateWeatherDataPoints( regionId, dateTime, tempcelcuis, dewPoint, windSpeedKmPh, windGustKmPh, windDirectionDegrees, windChill, visibilityMetric, null, // No UNIndex in Wunderground historical data entries precipitation, snow, pressure, humidityPercentage, conditionDescription, false); } } } catch (Exception e) { Logger.Error( $"WeatherDataMiner: MineHistoricWeatherValues(): Exception encountered while retrieving historical Weather Data figures for {regionSubUrl} between {historicStartDateTime} and {historicEndDateTime}.", "WeatherDataMiner.MineHistoricWeatherValues()", null, e); } }
/// <summary> /// Mine Or Calculate Relative Merit Data for emissions values between the given dates and store them in the database. /// The Relative Merit is a value between 0 (meaning best) and 1 (being worst). /// </summary> /// <param name="startDateTime">Optional StartDateTime. If not supplied, a default value will be used</param> /// <param name="endDateTime">Optional endDateTime. If not supplied, a default value will be used</param> /// <param name="regionWattTimeName">Abbreviation for the required region (e.g. "PJM"). See https://api.watttime.org/faq/#where </param> /// <param name="regionId">regionId of this region in the application's database</param> public void MineOrCalculateCarbonEmissionsRelativeMerit( string regionWattTimeName, int regionId) { var historicStartDateTime = DateTime.UtcNow.AddMinutes(-1); var historicEndDateTime = DateTime.UtcNow.AddMinutes(15); try { Logger.Information( $"Entering method for {regionWattTimeName} from WattTime URL {this.wattTimeApiUrl} with method of relative data retrieval / calculation of {this.RelativeMeritDataSource}.", "CarbonEmissionsMiner.MineOrCalculateHistoricRelativeMeritData()"); switch (this.RelativeMeritDataSource) { case "WattTime": var result = this.wattTimeEmissionsInteraction.GetCarbonEmissionsRelativeMeritResults( WattTimeV2ApiUrl, regionWattTimeName, WattTimeUsername, WattTimePassword ); if (result != null) { Logger.Information( $"Received result for RelativeMeritData for {regionWattTimeName} from WattTime. Inserting into the database", "CarbonEmissionsMiner.MineOrCalculateRelativeMeritData()"); // Insert results in the database using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { var dateTime = result.validUntil; double emissionsRelativeMerit = result.percent / 100; // Normalise percentage to a value between 0 and 1 _objectModel.InsertOrUpdateCarbonEmissionsRelativeMeritDataPoints( regionId, dateTime, emissionsRelativeMerit, null); } } else { Logger.Information( $"No result found when requesting RelativeMeritData for {regionWattTimeName} from WattTime at UTC: {DateTime.UtcNow}.e", "CarbonEmissionsMiner.MineOrCalculateRelativeMeritData()"); } break; case "CustomInternalCalculation": var calculatedHistoricRelativeMeritResults = this.CalculateHistoricRelativeMeritDataResults( regionId, historicStartDateTime, historicEndDateTime); // Insert results in the database using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { foreach (var res in calculatedHistoricRelativeMeritResults) { _objectModel.InsertOrUpdateCarbonEmissionsRelativeMeritDataPoints( regionId, res.Timestamp, res.EmissionsRelativeMerit, res.EmissionsRelativeMerit_Forcast); } } break; default: Logger.Information( $"No known defined method of MineOrCalculateRelativeMeritData supplied to method. Not mining or calculating Historic Relative Merit Data for this region ({regionWattTimeName})", "CarbonEmissionsMiner.MineOrCalculateRelativeMeritData()"); return; } } catch (Exception e) { Logger.Error( $"CarbonEmissionsMiner: MineOrCalculateRelativeMeritData(): Exception encountered while emissions Data figures for {regionWattTimeName} between {historicStartDateTime} and {historicEndDateTime}.", "CarbonEmissionsMiner.MineOrCalculateRelativeMeritData()", null, e); } }
/// <summary> /// Take a config element containing the data for a region to be mined, mine the data and save it in the database /// </summary> /// <param name="regionConfiguration"></param> /// <param name="wattTimeApiKeyOverride">Optional wattTimeApiKey to Override what's in the MinerConfigXML File</param> /// <param name="wundergroundApiKeyOverride">Optional WundergroundApiKey to Override what's in the MinerConfigXML File</param> public void MineRegionData(ApiMinerConfigLayoutRegion regionConfiguration, string wattTimeApiKeyOverride = null, string wundergroundApiKeyOverride = null) { var regionGroupingName = regionConfiguration.friendlyName; using (new TimedOperation( $"Beginning Mining of all data for Region {regionGroupingName}", "ApiDataMiner.MineRegionData()")) { // Mine the regions emissions if an emissions node was supplied int?emissionsRegionId = null; if (regionConfiguration.EmissionsMiningRegion != null) { var friendlyName = regionConfiguration.EmissionsMiningRegion.friendlyName; using ( new TimedOperation( $"Beginning Mining of emissions data for Region {friendlyName}", "ApiDataMiner.MineRegionData()")) { var timeZone = regionConfiguration.EmissionsMiningRegion.TimeZone; var regionLat = regionConfiguration.EmissionsMiningRegion.Latitude; var regionLong = regionConfiguration.EmissionsMiningRegion.Longitude; var regionWattTimeName = regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation; var wattTimeApiUrl = regionConfiguration.EmissionsMiningRegion.ApiUrl; string wattTimeApiKey = null; if (string.IsNullOrEmpty(wattTimeApiKeyOverride) || wattTimeApiKeyOverride.Equals("none")) { wattTimeApiKey = regionConfiguration.EmissionsMiningRegion.ApiKey; } else { wattTimeApiKey = wattTimeApiKeyOverride; } var selfThrottlingMethod = regionConfiguration.EmissionsMiningRegion.SelfThrottlingMethod; var maxNumberOfCallsPerMinute = regionConfiguration.EmissionsMiningRegion.MaxNumberOfCallsPerMinute; var historicStartDateTime = DateTime.UtcNow.AddDays(-15); var historicEndDateTime = DateTime.UtcNow.AddDays(1); var forecastStartDateTime = DateTime.UtcNow.AddDays(-2); var forecastEndDateTime = DateTime.UtcNow.AddDays(10); if (!string.IsNullOrEmpty(wattTimeApiKey) && !wattTimeApiKey.Equals("none")) { Logger.Information( $"About to add Emissions Region and Mine Carbon Emissions Data for {regionWattTimeName} from WattTime URL {wattTimeApiUrl} from {historicStartDateTime} to {historicEndDateTime} for historic data and insert them into the database", "ApiDataMiner.MineRegionData()"); using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { emissionsRegionId = _objectModel.AddEmissionsRegion(friendlyName, timeZone, regionLat, regionLong, regionWattTimeName) .EmissionsRegionID; CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner( wattTimeApiUrl, wattTimeApiKey, selfThrottlingMethod, this.DatabaseConnectionString, maxNumberOfCallsPerMinute); // Mine Recent Actual Data carbonEmissionsMiner.MineHistoricCarbonResults( historicStartDateTime, historicEndDateTime, regionWattTimeName, (int)emissionsRegionId); // Mine Forecast Data carbonEmissionsMiner.MineForecastMarginalCarbonResults( forecastStartDateTime, forecastEndDateTime, regionWattTimeName, (int)emissionsRegionId); } } else { Logger.Information( $"No WattTime Api Key was specified. Skipping this region for Emissions.", "RunAsync()"); } } } // Mine the regions weather if a weather node was supplied int?weatherRegionId = null; if (regionConfiguration.WeatherMiningRegion != null) { var friendlyName = regionConfiguration.WeatherMiningRegion.friendlyName; using ( new TimedOperation( $"Beginning Mining of weather data for Region {friendlyName}", "ApiDataMiner.MineRegionData()")) { var timeZone = regionConfiguration.WeatherMiningRegion.TimeZone; var regionLat = regionConfiguration.WeatherMiningRegion.Latitude; var regionLong = regionConfiguration.WeatherMiningRegion.Longitude; var weatherRegionWundergroundSubUrl = regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl; var wundergroundApiUrl = regionConfiguration.WeatherMiningRegion.ApiUrl; string wundergroundApiKey = null; if (string.IsNullOrEmpty(wundergroundApiKeyOverride) || wundergroundApiKeyOverride.Equals("none")) { wundergroundApiKey = regionConfiguration.WeatherMiningRegion.ApiKey; } else { wundergroundApiKey = wundergroundApiKeyOverride; } var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod; var maxNumberOfCallsPerMinute = regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute; var historicStartDateTime = DateTime.UtcNow.AddDays(-1); var historicEndDateTime = DateTime.UtcNow.AddDays(1); if (!string.IsNullOrEmpty(wundergroundApiKey) && !wundergroundApiKey.Equals("none")) { Logger.Information( $"About to add Emissions Region and Mine Carbon Emissions Data for {friendlyName} from Wunderground URL {weatherRegionWundergroundSubUrl} from {historicStartDateTime} to {historicEndDateTime} for historic data and insert them into the database", "ApiDataMiner.MineRegionData()"); using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { weatherRegionId = _objectModel.AddWeatherRegion( friendlyName, timeZone, regionLat, regionLong, weatherRegionWundergroundSubUrl).WeatherRegionID; WeatherDataMiner weatherDataMiner = new WeatherDataMiner( wundergroundApiUrl, wundergroundApiKey, selfThrottlingMethod, this.DatabaseConnectionString, maxNumberOfCallsPerMinute); switch (regionConfiguration.WeatherMiningRegion.MiningMethod) { case "GPS": // Mine Recent Actual Data weatherDataMiner.MineHistoricWeatherValues( historicStartDateTime, historicEndDateTime, regionLat, regionLong, (int)weatherRegionId); // Mine Forecast Data weatherDataMiner.MineTenDayHourlyForecastWeatherValues( regionLat, regionLong, (int)weatherRegionId); break; case "WundergroundPageSubUrl": default: // Mine Recent Actual Data weatherDataMiner.MineHistoricWeatherValues( historicStartDateTime, historicEndDateTime, weatherRegionWundergroundSubUrl, (int)weatherRegionId); // Mine Forecast Data weatherDataMiner.MineTenDayHourlyForecastWeatherValues( weatherRegionWundergroundSubUrl, (int)weatherRegionId); break; } } } else { Logger.Information( $"No Wunderground Api Key was specified. Skipping this region for Weather.", "RunAsync()"); } } } // Group the Emissions and Weather regions if they haven't already been grouped using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { _objectModel.AddMarketWeatherEmissionsRegionMapping( regionGroupingName, null, weatherRegionId, emissionsRegionId); } } }
private void InsertHistoricWeatherValuesIntoDatabase(int regionId, List <Observation> results) { using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { foreach (var res in results) { var dateTime = res.observationDateTime; double tempcelcuis; double.TryParse(res.tempm, out tempcelcuis); double dewPoint; double.TryParse(res.dewptm, out dewPoint); double humidityPercentage; double.TryParse(res.hum, out humidityPercentage); double windSpeedKmPh; double.TryParse(res.wspdm, out windSpeedKmPh); double windGustKmPh; double.TryParse(res.wgustm, out windGustKmPh); double windDirectionDegrees; double.TryParse(res.wdird, out windDirectionDegrees); double windChill; double.TryParse(res.windchillm, out windChill); double visibilityMetric; double.TryParse(res.vism, out visibilityMetric); double snow; double.TryParse(res.snow, out snow); double pressure; double.TryParse(res.pressurem, out pressure); double precipitation; double.TryParse(res.precipm, out precipitation); var conditionDescription = res.conds ?? string.Empty; // Clean up "-9999" values that sometimes come back if (windSpeedKmPh < 0) { windSpeedKmPh = -1; } if (windGustKmPh < 0) { windGustKmPh = -1; } _objectModel.InsertOrUpdateWeatherDataPoints( regionId, dateTime, tempcelcuis, dewPoint, windSpeedKmPh, windGustKmPh, windDirectionDegrees, windChill, visibilityMetric, null, // No UNIndex in Wunderground historical data entries precipitation, snow, pressure, humidityPercentage, conditionDescription, false); } } }
public void Initialize() { this.objectModel = new SmartEnergyOM(this.databaseConnectionString); }
public void TestMineHistoricSystemWideCarbonResults() { // Arrange var startDateTime = DateTime.Now.AddDays(-10); var endDateTime = DateTime.Now.AddDays(-1); var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl"); var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey"); string wattTimeApiV2Url = CloudConfigurationManager.GetSetting("WattTimeApiV2Url"); string WattTimeUsername = CloudConfigurationManager.GetSetting("WattTimeUsername"); string WattTimePassword = CloudConfigurationManager.GetSetting("WattTimePassword"); string WattTimeEmail = CloudConfigurationManager.GetSetting("WattTimeEmail"); string WattTimeOrganization = CloudConfigurationManager.GetSetting("WattTimeOrganization"); var selfThrottlingMethod = "AzureTableStorageCallRecollection"; var maxNumberOfCallsPerMinute = 9; List <WattTimeBalancingAuthorityInformation> regionsToMine = new List <WattTimeBalancingAuthorityInformation>(); regionsToMine.Add(new WattTimeBalancingAuthorityInformation("ERCOT", "US_ERCOT", "Central Standard Time", 32.79878236662912, -96.77856445062508)); foreach (var region in regionsToMine) { int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddEmissionsRegion(region.smartGridRegionName, region.timeZone, region.regionLat, region.regionLong) .EmissionsRegionID; _objectModel.AddMarketWeatherEmissionsRegionMapping( region.smartGridRegionName, null, null, regionId); } var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute); CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner( wattTimeApiUrl, wattTimeApiKey, wattTimeApiV2Url, WattTimeUsername, WattTimePassword, WattTimeEmail, WattTimeOrganization, selfThrottlingMethod, databaseConnectionString, maxNumberOfCallsPerMinute, wattTimeInteraction); // Act carbonEmissionsMiner.MineHistoricSystemWideCarbonResults( startDateTime, endDateTime, region.regionWattTimeName, regionId); // Assert // Verify that each data point has been recorded in the database var results = wattTimeInteraction.GetGenerationMixAndSystemWideEmissionsResults( wattTimeApiUrl, region.regionWattTimeName, startDateTime, endDateTime, null, wattTimeApiKey); using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { foreach (var result in results) { var dataPoint = _objectModel.FindCarbonEmissionsDataPoint(regionId, result.timestamp); Assert.IsNotNull(dataPoint); } } } }
public void TestMineForecastMarginalCarbonResults() { // Arrange var regionWattTimeName = "PJM"; var smartGridRegionName = "PJM"; var timeZone = "Eastern Standard Time"; var regionLat = 40.348444276169; var regionLong = -74.6428556442261; var startDateTime = DateTime.UtcNow.AddDays(-2); var endDateTime = DateTime.UtcNow.AddDays(10); var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl"); var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey"); var selfThrottlingMethod = "AzureTableStorageCallRecollection"; var maxNumberOfCallsPerMinute = 9; int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddEmissionsRegion(smartGridRegionName, timeZone, regionLat, regionLong) .EmissionsRegionID; } var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute); CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner( wattTimeApiUrl, wattTimeApiKey, selfThrottlingMethod, databaseConnectionString, maxNumberOfCallsPerMinute, wattTimeInteraction); // Act carbonEmissionsMiner.MineForecastMarginalCarbonResults( startDateTime, endDateTime, regionWattTimeName, regionId); // Assert // Verify that each data point has been recorded in the database var results = wattTimeInteraction.GetForecastMarginalCarbonResults( wattTimeApiUrl, regionWattTimeName, startDateTime, endDateTime, null, wattTimeApiKey); using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { foreach (var result in results) { if (result.marginal_carbon.value != null) { var dataPoint = _objectModel.FindCarbonEmissionsDataPoints(regionId, result.timestamp); Assert.IsNotNull(dataPoint); Assert.AreEqual( wattTimeInteraction.ConvertLbsPerMWhTo_GCo2PerkWh((double)result.marginal_carbon.value), dataPoint.MarginalCO2Intensity_gCO2kWh); } } } }
/// <summary> /// Calculate the Relative Merit of Marginal Carbon Emissions Values which are present in the database between the given startDateTime and endDateTime. /// The Relative Merit is a value between 0 (meaning best) and 1 (being worst). /// </summary> /// <param name="regionId"></param> /// <param name="startDateTime"></param> /// <param name="endDateTime"></param> /// <returns>List of MarginalCarbonResult.Result representing the Relative Merit of the corresponding Marginal Carbon Emissions Values</returns> public List <EmissionsRelativeMeritDatapoint> CalculateHistoricRelativeMeritDataResults(int regionId, DateTime startDateTime, DateTime endDateTime) { /*** Implement your custom logic here to calculate relative merit data ***/ /* Here is a sample method provided here which simply calculates it's comparison to a one week rolling average for the data */ var results = new List <EmissionsRelativeMeritDatapoint>(); using (var objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { // Get datapoints on which to calculate relative merit var carbonResultsOnWhichToCalculateRelativeMetit = objectModel.FindCarbonEmissionsDataPoints(regionId, startDateTime, endDateTime); foreach (var datapoint in carbonResultsOnWhichToCalculateRelativeMetit) { try { // Get the last week of data var startDateOfOneWeekRollingAverage = datapoint.DateTimeUTC.AddDays(-7); var endDateOfOneWeekRollingAverage = datapoint.DateTimeUTC.AddHours(1); var emissionsDataPointsWithinWindow = objectModel.FindCarbonEmissionsDataPoints(regionId, startDateOfOneWeekRollingAverage, endDateOfOneWeekRollingAverage).Where(a => a.MarginalCO2Intensity_gCO2kWh != null); // Calculate where this datapoints falls within the range of the last week if (emissionsDataPointsWithinWindow.Any()) { double?relativeMerit = null; double?relativeMerit_Forecast = null; if (datapoint.MarginalCO2Intensity_gCO2kWh != null) { var maxValue = emissionsDataPointsWithinWindow.Max(a => a.MarginalCO2Intensity_gCO2kWh); var minValue = emissionsDataPointsWithinWindow.Min(a => a.MarginalCO2Intensity_gCO2kWh); var range = (double)(maxValue - minValue); relativeMerit = (datapoint.MarginalCO2Intensity_gCO2kWh - minValue) / range; // One specicial check: 0 should be reserved for zero emissions. Set anything above zero to .2 if ((relativeMerit < .2) && (datapoint.MarginalCO2Intensity_gCO2kWh > 0)) { relativeMerit = .2; } } if (datapoint.MarginalCO2Intensity_Forcast_gCO2kWh != null) { var maxValue = emissionsDataPointsWithinWindow.Max(a => a.MarginalCO2Intensity_Forcast_gCO2kWh); var minValue = emissionsDataPointsWithinWindow.Min(a => a.MarginalCO2Intensity_Forcast_gCO2kWh); var range = (double)(maxValue - minValue); relativeMerit_Forecast = (datapoint.MarginalCO2Intensity_Forcast_gCO2kWh - minValue) / range; // One specicial check: 0 should be reserved for zero emissions. Set anything above zero to .2 if ((relativeMerit_Forecast < .2) && (datapoint.MarginalCO2Intensity_Forcast_gCO2kWh > 0)) { relativeMerit_Forecast = .2; } } if ((relativeMerit != null) || (relativeMerit_Forecast != null)) { var emissionsRelativeMeritDataResult = new EmissionsRelativeMeritDatapoint { EmissionsRegionID = regionId, Timestamp = datapoint.DateTimeUTC, EmissionsRelativeMerit = relativeMerit, EmissionsRelativeMerit_Forcast = relativeMerit_Forecast }; results.Add(emissionsRelativeMeritDataResult); } } } catch (Exception e) { Logger.Error( $"CarbonEmissionsMiner: CalculateHistoricRelativeMeritDataResults(): Exception encountered Calculate the Merit of datapoint at {datapoint.DateTimeUTC} for RegionId {datapoint.EmissionsRegionID}.", "CarbonEmissionsMiner.CalculateHistoricRelativeMeritDataResults()", null, e); } } } return(results); }
public void TestMineHistoricMarginalCarbonResults() { // Arrange var startDateTime = DateTime.Now.AddDays(-10); var endDateTime = DateTime.Now.AddDays(-9); var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl"); var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey"); var selfThrottlingMethod = "AzureTableStorageCallRecollection"; var maxNumberOfCallsPerMinute = 9; List <WattTimeBalancingAuthorityInformation> regionsToMine = new List <WattTimeBalancingAuthorityInformation> { new WattTimeBalancingAuthorityInformation( "PJM", "US_PJM", "Eastern Standard Time", 40.348444276169, -74.6428556442261) }; foreach (var region in regionsToMine) { int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddEmissionsRegion(region.smartGridRegionName, region.timeZone, region.regionLat, region.regionLong) .EmissionsRegionID; _objectModel.AddMarketWeatherEmissionsRegionMapping( region.smartGridRegionName, null, null, regionId); } var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute); CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner( wattTimeApiUrl, wattTimeApiKey, selfThrottlingMethod, databaseConnectionString, maxNumberOfCallsPerMinute, wattTimeInteraction); // Act carbonEmissionsMiner.MineHistoricMarginalCarbonResults( startDateTime, endDateTime, region.regionWattTimeName, regionId); // Assert // Verify that each data point has been recorded in the database var results = wattTimeInteraction.GetObservedMarginalCarbonResults( wattTimeApiUrl, region.regionWattTimeName, startDateTime, endDateTime, null, wattTimeApiKey); using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { foreach (var result in results) { var dataPoint = _objectModel.FindCarbonEmissionsDataPoints(regionId, result.timestamp); Assert.IsNotNull(dataPoint); } } } }
public void TestMineHistoricMarginalCarbonResults() { // Arrange var startDateTime = DateTime.Now.AddDays(-10); var endDateTime = DateTime.Now.AddDays(-1); var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl"); var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey"); string wattTimeApiV2Url = CloudConfigurationManager.GetSetting("WattTimeApiV2Url"); string WattTimeUsername = CloudConfigurationManager.GetSetting("WattTimeUsername"); string WattTimePassword = CloudConfigurationManager.GetSetting("WattTimePassword"); string WattTimeEmail = CloudConfigurationManager.GetSetting("WattTimeEmail"); string WattTimeOrganization = CloudConfigurationManager.GetSetting("WattTimeOrganization"); var selfThrottlingMethod = "AzureTableStorageCallRecollection"; var maxNumberOfCallsPerMinute = 9; List <WattTimeBalancingAuthorityInformation> regionsToMine = new List <WattTimeBalancingAuthorityInformation>(); regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PSEI", "US_PugetSoundEnergy", "Pacific Standard Time", 47.68009593341535, -122.11638450372567)); regionsToMine.Add(new WattTimeBalancingAuthorityInformation("BPA", "US_BPA", "Pacific Standard Time", 40.348444276169, -74.6428556442261)); regionsToMine.Add(new WattTimeBalancingAuthorityInformation("CAISO", "US_CAISO", "Pacific Standard Time", 41.7324, -123.409423)); regionsToMine.Add(new WattTimeBalancingAuthorityInformation("ERCOT", "US_ERCOT", "Central Standard Time", 32.79878236662912, -96.77856445062508)); regionsToMine.Add(new WattTimeBalancingAuthorityInformation("ISONE", "US_ISONewEngland", "Eastern Standard Time", 42.70864591994315, -72.16918945062508)); regionsToMine.Add(new WattTimeBalancingAuthorityInformation("MISO", "US_UpperMidwestISO", "Central Standard Time", 41.91853269857261, -93.55193137872567)); regionsToMine.Add(new WattTimeBalancingAuthorityInformation("SPP", "US_SouthwesternPublicServiceISO", "Eastern Standard Time", 34.41133502036136, -103.19243430841317)); regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PJM", "US_PJM", "Eastern Standard Time", 40.348444276169, -74.6428556442261)); regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PJM_ATLANTIC", "US_PJM_ATLANTIC", "Eastern Standard Time", 40.566564, -76.98465597395705)); regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PJM_SOUTH", "US_PJM_SOUTH", "Eastern Standard Time", 37.44276433719146, -76.87479269270705)); regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PJM_WEST", "US_PJM_WEST", "Eastern Standard Time", 38.69484915602888, -85.11453878645705)); foreach (var region in regionsToMine) { int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddEmissionsRegion(region.smartGridRegionName, region.timeZone, region.regionLat, region.regionLong) .EmissionsRegionID; _objectModel.AddMarketWeatherEmissionsRegionMapping( region.smartGridRegionName, null, null, regionId); } var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute); CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner( wattTimeApiUrl, wattTimeApiKey, wattTimeApiV2Url, WattTimeUsername, WattTimePassword, WattTimeEmail, WattTimeOrganization, selfThrottlingMethod, databaseConnectionString, maxNumberOfCallsPerMinute, wattTimeInteraction); // Act carbonEmissionsMiner.MineHistoricMarginalCarbonResults( startDateTime, endDateTime, region.regionWattTimeName, regionId); // Assert // Verify that each data point has been recorded in the database var results = wattTimeInteraction.GetObservedMarginalCarbonResults( wattTimeApiV2Url, region.regionWattTimeName, WattTimeUsername, WattTimePassword, startDateTime, endDateTime, null, null); using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { foreach (var result in results) { var dataPoint = _objectModel.FindCarbonEmissionsDataPoint(regionId, result.point_time); Assert.IsNotNull(dataPoint); } } } }
/// <summary> /// Retrieve forecast weather data from the Wunderground Weather service for the given regionSubUrl (e.g. CA/San_Francisco) and add it to the database /// </summary> /// <param name="regionSubUrl">Sub Url of the region on the Wunderground weather API e.g. CA/San_Francisco</param> /// <param name="regionId">regionId of this region in the application's database</param> public void MineTenDayHourlyForecastWeatherValues(string regionSubUrl, int regionId) { try { var results = this.wundergroundWeatherInteraction.GetTenDayHourlyForecastWeatherData( this.wundergroundApiUrl, regionSubUrl, this.wundergroundApiKey); Logger.Information( $"Received {results.Count} TenDayHourlyForecastWeatherValues Results for region with SubUrl {regionSubUrl} from Wunderground. Inserting them into the database", "WeatherDataMiner.MineTenDayHourlyForecastWeatherValues()"); // Insert results in the database using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { foreach (var res in results) { var dateTime = res.observationDateTime; double tempcelcuis; double.TryParse(res.temp.metric, out tempcelcuis); double dewPoint; double.TryParse(res.dewpoint.metric, out dewPoint); double windSpeedKmPh; double.TryParse(res.wspd.metric, out windSpeedKmPh); double windDirectionDegrees; double.TryParse(res.wdir.degrees, out windDirectionDegrees); double windChill; double.TryParse(res.windchill.metric, out windChill); double uvIndex; double.TryParse(res.uvi, out uvIndex); double snow; double.TryParse(res.snow.metric, out snow); double pressure; double.TryParse(res.mslp.metric, out pressure); double humidityPercentage; double.TryParse(res.humidity, out humidityPercentage); double precipitation; double.TryParse(res.qpf.metric, out precipitation); var conditionDescription = res.condition; _objectModel.InsertOrUpdateWeatherDataPoints( regionId, dateTime, tempcelcuis, dewPoint, windSpeedKmPh, null, // No wind gusts in Wunderground forecasts windDirectionDegrees, windChill, null, // No visibilty in Wunderground forecasts uvIndex, precipitation, snow, pressure, humidityPercentage, conditionDescription, true); } } } catch (Exception e) { Logger.Error( $"WeatherDataMiner: MineHistoricWeatherValues(): Exception encountered while retrieving historical Weather Data figures for {regionSubUrl}.", "WeatherDataMiner.MineTenDayHourlyForecastWeatherValues()", null, e); } }
public void TestParseMinerSettingsFileAndMineData() { // Arrange var ConfigPath = @".\ApiDataMinerConfigs\ApiDataMinerConfigs.xml"; // Act var apiDataMiner = new ApiDataMiner(databaseConnectionString); apiDataMiner.ParseMinerSettingsFileAndMineData(ConfigPath); // Assert using (var streamReader = new StreamReader(ConfigPath)) { var xmlSerializer = new XmlSerializer(typeof(ApiMinerConfigLayout)); var minerConfigs = (ApiMinerConfigLayout)xmlSerializer.Deserialize(streamReader); foreach (var regionConfiguration in minerConfigs.Regions) { // Verify emissions were mined successfully for each region in the Config File if (regionConfiguration.EmissionsMiningRegion != null) { var emissionsRegionName = regionConfiguration.EmissionsMiningRegion.friendlyName; var timeZone = regionConfiguration.EmissionsMiningRegion.TimeZone; var regionLat = regionConfiguration.EmissionsMiningRegion.Latitude; var regionLong = regionConfiguration.EmissionsMiningRegion.Longitude; var regionWattTimeName = regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation; var wattTimeApiUrl = regionConfiguration.EmissionsMiningRegion.ApiUrl; var wattTimeApiKey = regionConfiguration.EmissionsMiningRegion.ApiKey; var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod; var maxNumberOfCallsPerMinute = regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute; var startDateTime = DateTime.UtcNow.AddDays(-2); var endDateTime = DateTime.UtcNow.AddDays(10); var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute); var results = wattTimeInteraction.GetObservedMarginalCarbonResults( wattTimeApiUrl, regionWattTimeName, startDateTime, endDateTime, null, wattTimeApiKey); int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddEmissionsRegion(emissionsRegionName, timeZone, regionLat, regionLong) .EmissionsRegionID; } using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { foreach (var result in results) { var dataPoint = _objectModel.FindCarbonEmissionsDataPoints(regionId, result.timestamp); Assert.IsNotNull(dataPoint); } } } // Verify weather was mined successfully for each region in the Config File if (regionConfiguration.WeatherMiningRegion != null) { var emissionsRegionName = regionConfiguration.WeatherMiningRegion.friendlyName; var timeZone = regionConfiguration.WeatherMiningRegion.TimeZone; var regionLat = regionConfiguration.WeatherMiningRegion.Latitude; var regionLong = regionConfiguration.WeatherMiningRegion.Longitude; var weatherRegionWundergroundSubUrl = regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl; var wundergroundApiUrl = regionConfiguration.WeatherMiningRegion.ApiUrl; var wundergroundApiKey = regionConfiguration.WeatherMiningRegion.ApiKey; var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod; var maxNumberOfCallsPerMinute = regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute; var wundergroundWeatherInteraction = new WundergroundWeatherInteraction( selfThrottlingMethod, maxNumberOfCallsPerMinute); var results = wundergroundWeatherInteraction.GetTenDayHourlyForecastWeatherData( wundergroundApiUrl, weatherRegionWundergroundSubUrl, wundergroundApiKey); int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.AddWeatherRegion(emissionsRegionName, timeZone, regionLat, regionLong) .WeatherRegionID; } using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { foreach (var result in results) { var dataPoint = _objectModel.FindWeatherDataPoints( regionId, result.observationDateTime); Assert.IsNotNull(dataPoint); } } } } } }
public void TestCalculateHistoricRelativeMeritDataResults() { // Arrange var smartGridRegionName = "US_PJM"; var startDateTime = DateTime.UtcNow.AddDays(-10); var endDateTime = DateTime.UtcNow.AddDays(-9); var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl"); var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey"); string wattTimeApiV2Url = CloudConfigurationManager.GetSetting("WattTimeApiV2Url"); string WattTimeUsername = CloudConfigurationManager.GetSetting("WattTimeUsername"); string WattTimePassword = CloudConfigurationManager.GetSetting("WattTimePassword"); string WattTimeEmail = CloudConfigurationManager.GetSetting("WattTimeEmail"); string WattTimeOrganization = CloudConfigurationManager.GetSetting("WattTimeOrganization"); var selfThrottlingMethod = "AzureTableStorageCallRecollection"; var maxNumberOfCallsPerMinute = 200; var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute); var carbonEmissionsMiner = new CarbonEmissionsMiner( wattTimeApiUrl, wattTimeApiKey, wattTimeApiV2Url, WattTimeUsername, WattTimePassword, WattTimeEmail, WattTimeOrganization, selfThrottlingMethod, databaseConnectionString, maxNumberOfCallsPerMinute, wattTimeInteraction); int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.FindEmissionsRegion(smartGridRegionName).EmissionsRegionID; } Assert.IsNotNull(regionId, "Couldn't find specified emissions region in the database"); // Act var results = carbonEmissionsMiner.CalculateHistoricRelativeMeritDataResults( regionId, startDateTime, endDateTime ); // Assert foreach (var item in results) { if (item.EmissionsRelativeMerit != null) { Assert.IsTrue(item.EmissionsRelativeMerit >= 0, "item.EmissionsRelativeMerit >= 0"); Assert.IsTrue(item.EmissionsRelativeMerit <= 1, "item.EmissionsRelativeMerit <= 1"); } if (item.EmissionsRelativeMerit_Forcast != null) { Assert.IsTrue(item.EmissionsRelativeMerit_Forcast >= 0, "item.EmissionsRelativeMerit_Forcast >= 0"); Assert.IsTrue(item.EmissionsRelativeMerit_Forcast <= 1, "item.EmissionsRelativeMerit_Forcast <= 1"); } } }
/// <summary> /// Take a config element containing the data for a region to be mined, mine the data and save it in the database /// </summary> /// <param name="regionConfiguration"></param> public void MineRegionData(ApiMinerConfigLayoutRegion regionConfiguration) { var regionGroupingName = regionConfiguration.friendlyName; using (new TimedOperation( $"Beginning Mining of all data for Region {regionGroupingName}", "ApiDataMiner.MineRegionData()")) { // Mine the regions emissions if an emissions node was supplied int?emissionsRegionId = null; if (regionConfiguration.EmissionsMiningRegion != null) { var friendlyName = regionConfiguration.EmissionsMiningRegion.friendlyName; using ( new TimedOperation( $"Beginning Mining of emissions data for Region {friendlyName}", "ApiDataMiner.MineRegionData()")) { var timeZone = regionConfiguration.EmissionsMiningRegion.TimeZone; var regionLat = regionConfiguration.EmissionsMiningRegion.Latitude; var regionLong = regionConfiguration.EmissionsMiningRegion.Longitude; var regionWattTimeName = regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation; var wattTimeApiUrl = regionConfiguration.EmissionsMiningRegion.ApiUrl; var wattTimeApiKey = regionConfiguration.EmissionsMiningRegion.ApiKey; var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod; var maxNumberOfCallsPerMinute = regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute; var historicStartDateTime = DateTime.UtcNow.AddDays(-15); var historicEndDateTime = DateTime.UtcNow.AddDays(1); var forecastStartDateTime = DateTime.UtcNow.AddDays(-2); var forecastEndDateTime = DateTime.UtcNow.AddDays(10); using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { emissionsRegionId = _objectModel.AddEmissionsRegion(friendlyName, timeZone, regionLat, regionLong) .EmissionsRegionID; CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner( wattTimeApiUrl, wattTimeApiKey, selfThrottlingMethod, this.DatabaseConnectionString, maxNumberOfCallsPerMinute); // Mine Recent Actual Data carbonEmissionsMiner.MineHistoricCarbonResults( historicStartDateTime, historicEndDateTime, regionWattTimeName, (int)emissionsRegionId); // Mine Forecast Data carbonEmissionsMiner.MineForecastMarginalCarbonResults( forecastStartDateTime, forecastEndDateTime, regionWattTimeName, (int)emissionsRegionId); } } } // Mine the regions emissions if an emissions node was supplied int?weatherRegionId = null; if (regionConfiguration.WeatherMiningRegion != null) { var friendlyName = regionConfiguration.WeatherMiningRegion.friendlyName; using ( new TimedOperation( $"Beginning Mining of weather data for Region {friendlyName}", "ApiDataMiner.MineRegionData()")) { var timeZone = regionConfiguration.WeatherMiningRegion.TimeZone; var regionLat = regionConfiguration.WeatherMiningRegion.Latitude; var regionLong = regionConfiguration.WeatherMiningRegion.Longitude; var weatherRegionWundergroundSubUrl = regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl; var wundergroundApiUrl = regionConfiguration.WeatherMiningRegion.ApiUrl; var wundergroundApiKey = regionConfiguration.WeatherMiningRegion.ApiKey; var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod; var maxNumberOfCallsPerMinute = regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute; var historicStartDateTime = DateTime.UtcNow.AddDays(-1); var historicEndDateTime = DateTime.UtcNow.AddDays(1); using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { weatherRegionId = _objectModel.AddWeatherRegion( friendlyName, timeZone, regionLat, regionLong, weatherRegionWundergroundSubUrl).WeatherRegionID; WeatherDataMiner weatherDataMiner = new WeatherDataMiner( wundergroundApiUrl, wundergroundApiKey, selfThrottlingMethod, this.DatabaseConnectionString, maxNumberOfCallsPerMinute); // Mine Recent Actual Data weatherDataMiner.MineHistoricWeatherValues( historicStartDateTime, historicEndDateTime, weatherRegionWundergroundSubUrl, (int)weatherRegionId); // Mine Forecast Data weatherDataMiner.MineTenDayHourlyForecastWeatherValues( weatherRegionWundergroundSubUrl, (int)weatherRegionId); } } } // Group the Emissions and Weather regions if they haven't already been grouped using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString)) { _objectModel.AddMarketWeatherEmissionsRegionMapping( regionGroupingName, null, weatherRegionId, emissionsRegionId); } } }
public void TestMineOrCalculateCarbonEmissionsRelativeMerit_CustomInternalCalculation() { // Arrange var regionWattTimeName = "PJM"; var smartGridRegionName = "US_PJM"; var startDateTime = DateTime.UtcNow.AddDays(-10); var endDateTime = DateTime.UtcNow.AddDays(-9); var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl"); var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey"); string wattTimeApiV2Url = CloudConfigurationManager.GetSetting("WattTimeApiV2Url"); string WattTimeUsername = CloudConfigurationManager.GetSetting("WattTimeUsername"); string WattTimePassword = CloudConfigurationManager.GetSetting("WattTimePassword"); string WattTimeEmail = CloudConfigurationManager.GetSetting("WattTimeEmail"); string WattTimeOrganization = CloudConfigurationManager.GetSetting("WattTimeOrganization"); var selfThrottlingMethod = "AzureTableStorageCallRecollection"; var maxNumberOfCallsPerMinute = 200; var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute); CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner( wattTimeApiUrl, wattTimeApiKey, wattTimeApiV2Url, WattTimeUsername, WattTimePassword, WattTimeEmail, WattTimeOrganization, selfThrottlingMethod, databaseConnectionString, maxNumberOfCallsPerMinute, wattTimeInteraction, "CustomInternalCalculation"); int regionId; using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { regionId = _objectModel.FindEmissionsRegion(smartGridRegionName).EmissionsRegionID; } Assert.IsNotNull(regionId, "Couldn't find specified emissions region in the database"); // Act carbonEmissionsMiner.MineOrCalculateCarbonEmissionsRelativeMerit( regionWattTimeName, regionId); // Assert // Verify that each data point has been recorded in the database var results = carbonEmissionsMiner.CalculateHistoricRelativeMeritDataResults( regionId, startDateTime, endDateTime); foreach (var result in results) { using (var _objectModel = new SmartEnergyOM(databaseConnectionString)) { var dataPoint = _objectModel.FindCarbonEmissionsRelativeMeritDataPoints(regionId, result.Timestamp.AddMinutes(-5), result.Timestamp.AddMinutes(15)); Assert.IsNotNull(dataPoint); } } }