public void TestMineHistoricWeatherValues()
{
// Arrange
var regionSubUrl = "us/nj/princeton";
string smartGridRegionName = "PJM";
var timeZone = "Eastern Standard Time";
var regionLat = 40.348444276169;
var regionLong = -74.6428556442261;
var startDateTime = DateTime.Now.AddDays(-3);
var endDateTime = DateTime.Now.AddDays(-1);
var wundergroundApiUrl = CloudConfigurationManager.GetSetting("WundergroundApiUrl");
var wundergroundApiKey = CloudConfigurationManager.GetSetting("WundergroundApiKey");
var selfThrottlingMethod = "AzureTableStorageCallRecollection";
var maxNumberOfCallsPerMinute = 5;
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddWeatherRegion(smartGridRegionName, timeZone, regionLat, regionLong).WeatherRegionID;
}
var wundergroundWeatherInteraction =
new WundergroundWeatherInteraction(
selfThrottlingMethod,
maxNumberOfCallsPerMinute);
WeatherDataMiner weatherDataMiner = new WeatherDataMiner(
wundergroundApiUrl,
wundergroundApiKey,
selfThrottlingMethod,
databaseConnectionString,
maxNumberOfCallsPerMinute,
wundergroundWeatherInteraction);
// Act
weatherDataMiner.MineHistoricWeatherValues(startDateTime, endDateTime, regionSubUrl, regionId);
// Assert
// Verify that each data point has been recorded in the database
var results = wundergroundWeatherInteraction.GetHistoricWeatherData(
wundergroundApiUrl,
regionSubUrl,
wundergroundApiKey,
startDateTime,
endDateTime);
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
foreach (var result in results)
{
var dataPoint = _objectModel.FindWeatherDataPoints(regionId, result.observationDateTime);
Assert.IsNotNull(dataPoint);
}
}
}
public IHttpActionResult Get([FromUri] string id, [FromUri] DateTime?startDateTime = null, [FromUri] DateTime?endDateTime = null,
[FromUri] double dateTimeFlexabilityInMinutes = 0)
{
try
{
// Set default start and end datetimes
string emissionsRegionFriendlyName = id.IsNullOrWhiteSpace() ? "US_PJM" : id;
var startDateTimeProcessed = startDateTime ?? DateTime.UtcNow.AddHours(-3);
var endDateTimeProcessed = endDateTime ?? DateTime.UtcNow.AddHours(1);
// Query database
var databaseConnectionString = CloudConfigurationManager.GetSetting("SQLAzureDatabaseEntityFrameworkConnectionString");
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
var regionId = _objectModel.FindEmissionsRegion(emissionsRegionFriendlyName).EmissionsRegionID;
var result = _objectModel.FindCarbonEmissionsRelativeMeritDataPoints(regionId, startDateTimeProcessed, endDateTimeProcessed, dateTimeFlexabilityInMinutes).ToList();
// Convert the final list to objects of the common data type in the SmartEnergyAzureDataTypes NuGet package
var webResult = this.ConvertToCarbonEmissionsRelativeMeritWebDataPoints(result);
if (result == null)
{
return(this.NotFound());
}
return(Ok(webResult));
}
}
catch (Exception e)
{
throw new Exception("Sorry - an exception occured executing the request");
}
}
public void TestMineHistoricWeatherValues_ByWundergroundLocationName()
{
// Arrange
var regionSubUrl = "ie/dublin";
var latitude = 53.3498;
var longtitude = -6.2603;
var smartGridRegionName = "Ireland";
var timeZone = "GMT Standard Time";
var startDateTime = DateTime.Now.AddDays(-10);
var endDateTime = DateTime.Now.AddDays(-1);
var wundergroundApiUrl = CloudConfigurationManager.GetSetting("WundergroundApiUrl");
var wundergroundApiKey = CloudConfigurationManager.GetSetting("WundergroundApiKey");
var selfThrottlingMethod = "AzureTableStorageCallRecollection";
var maxNumberOfCallsPerMinute = 5;
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddWeatherRegion(smartGridRegionName, timeZone, regionLat, regionLong, regionSubUrl).WeatherRegionID;
}
var wundergroundWeatherInteraction =
new WundergroundWeatherInteraction(
selfThrottlingMethod,
maxNumberOfCallsPerMinute);
WundergroundWeatherDataMiner weatherDataMiner = new WundergroundWeatherDataMiner(
wundergroundApiUrl,
wundergroundApiKey,
selfThrottlingMethod,
databaseConnectionString,
maxNumberOfCallsPerMinute,
wundergroundWeatherInteraction);
// Act
weatherDataMiner.MineHistoricWeatherValues(startDateTime, endDateTime, regionSubUrl, regionId);
// Assert
// Verify that each data point has been recorded in the database
var results = wundergroundWeatherInteraction.GetHistoricWeatherData(
wundergroundApiUrl,
regionSubUrl,
wundergroundApiKey,
startDateTime,
endDateTime);
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
foreach (var result in results)
{
var dataPoint = _objectModel.FindWeatherDataPoint(regionId, result.observationDateTime);
Assert.IsNotNull(dataPoint);
}
}
}
/// <summary>
/// Retrieve historic marginal emissions results between the given dates and store them in the database
/// </summary>
/// <param name="startDateTime">Optional StartDateTime. If not supplied, a default value will be used</param>
/// <param name="endDateTime">Optional endDateTime. If not supplied, a default value will be used</param>
/// <param name="regionWattTimeName">Abbreviation for the required region (e.g. "PJM"). See https://api.watttime.org/faq/#where </param>
/// <param name="regionId">regionId of this region in the application's database</param>
public void MineHistoricMarginalCarbonResults(
DateTime?startDateTime,
DateTime?endDateTime,
string regionWattTimeName,
int regionId)
{
var historicStartDateTime = startDateTime ?? DateTime.UtcNow.AddDays(-2);
var historicEndDateTime = endDateTime ?? DateTime.UtcNow.AddMinutes(15);
try
{
var results = this.wattTimeEmissionsInteraction.GetObservedMarginalCarbonResults(
wattTimeApiUrl,
regionWattTimeName,
historicStartDateTime,
historicEndDateTime,
null,
wattTimeApiKey);
Logger.Information(
$"Received {results.Count} HistoricMarginalCarbonResults Results for {regionWattTimeName} from WattTime. Inserting them into the database",
"CarbonEmissionsMiner.MineHistoricMarginalCarbonResults()");
// Insert results in the database
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
foreach (var res in results)
{
var dateTime = res.timestamp;
var marginalCarbon = res.marginal_carbon.value;
var units = res.marginal_carbon.units;
if (marginalCarbon != null)
{
var marginalCarbonMetric = units == "lb/MW"
? this.wattTimeEmissionsInteraction
.ConvertLbsPerMWhTo_GCo2PerkWh((double)marginalCarbon)
: marginalCarbon;
_objectModel.InsertOrUpdateCarbonEmissionsDataPoints(
regionId,
dateTime,
null,
false,
marginalCarbonMetric,
false);
}
}
}
}
catch (Exception e)
{
Logger.Error(
$"CarbonEmissionsMiner: MineHistoricMarginalCarbonResults(): Exception encountered while emissions Data figures for {regionWattTimeName} between {historicStartDateTime} and {historicEndDateTime}.",
"CarbonEmissionsMiner.MineHistoricMarginalCarbonResults()",
null,
e);
}
}
private void InsertWeatherValuesIntoDatabase(int regionId, List <HourlyDatum> results, bool areForecastRows = false)
{
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
foreach (var res in results)
{
var dateTime = res.dateTime;
_objectModel.InsertOrUpdateWeatherDataPoints(
regionId,
dateTime,
res.temperature,
res.dewPoint,
res.windSpeed,
null,
res.windBearing,
res.apparentTemperature,
res.visibility,
null,
null,
null,
res.pressure,
res.humidity,
res.summary,
areForecastRows);
}
}
}
/// <summary>
/// Retrieve forecast marginal emissions results between the given dates and store them in the database
/// </summary>
/// <param name="startDateTime">Optional StartDateTime. If not supplied, a default value will be used</param>
/// <param name="endDateTime">Optional endDateTime. If not supplied, a default value will be used</param>
/// <param name="regionWattTimeName">Abbreviation for the required region (e.g. "PJM"). See https://api.watttime.org/faq/#where </param>
/// <param name="regionId">regionId of this region in the application's database</param>
public void MineForecastMarginalCarbonResults(
DateTime?startDateTime,
DateTime?endDateTime,
string regionWattTimeName,
int regionId)
{
var historicStartDateTime = startDateTime ?? DateTime.UtcNow.AddDays(-2);
var historicEndDateTime = endDateTime ?? DateTime.UtcNow.AddMinutes(15);
try
{
Logger.Information(
$"Mining Forecast Marginal Carbon Results for {regionWattTimeName} from WattTime URL {this.wattTimeApiUrl}.",
"CarbonEmissionsMiner.MineForecastMarginalCarbonResults()");
var results = this.wattTimeEmissionsInteraction.GetForecastMarginalCarbonResults(
WattTimeV2ApiUrl,
regionWattTimeName,
WattTimeUsername,
WattTimePassword,
historicStartDateTime,
historicEndDateTime,
null);
Logger.Information(
$"Received {results.Count} ForecastMarginalCarbonResults Results for {regionWattTimeName} from WattTime. Inserting them into the database",
"CarbonEmissionsMiner.ForecastMarginalCarbonResults()");
// Insert results in the database
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
foreach (var res in results)
{
var dateTime = res.point_time;
var marginalCarbon = res.value;
var marginalCarbonMetric = this.wattTimeEmissionsInteraction
.ConvertLbsPerMWhTo_GCo2PerkWh((double)marginalCarbon);
_objectModel.InsertOrUpdateCarbonEmissionsDataPoints(
regionId,
dateTime,
null,
null,
null,
marginalCarbonMetric);
}
}
}
catch (Exception e)
{
Logger.Error(
$"CarbonEmissionsMiner: MineForecastMarginalCarbonResults(): Exception encountered while emissions Data figures for {regionWattTimeName} between {historicStartDateTime} and {historicEndDateTime}.",
"CarbonEmissionsMiner.MineForecastMarginalCarbonResults()",
null,
e);
}
}
/// <summary>
/// Mine Or Calculate Relative Merit Data for emissions values between the given dates and store them in the database.
/// The Relative Merit is a value between 0 (meaning best) and 1 (being worst).
/// </summary>
/// <param name="startDateTime">Optional StartDateTime. If not supplied, a default value will be used</param>
/// <param name="endDateTime">Optional endDateTime. If not supplied, a default value will be used</param>
/// <param name="regionWattTimeName">Abbreviation for the required region (e.g. "PJM"). See https://api.watttime.org/faq/#where </param>
/// <param name="regionId">regionId of this region in the application's database</param>
public void MineOrCalculateHistoricRelativeMeritData(
double latitude,
double longitude,
DateTime?startDateTime,
DateTime?endDateTime,
string regionWattTimeName,
int regionId)
{
var historicStartDateTime = startDateTime ?? DateTime.UtcNow.AddDays(-2);
var historicEndDateTime = endDateTime ?? DateTime.UtcNow.AddMinutes(15);
try
{
Logger.Information(
$"Entering method for {regionWattTimeName} from WattTime URL {this.wattTimeApiUrl} with method of relative data retrieval / calculation of {this.RelativeMeritDataSource}.",
"CarbonEmissionsMiner.MineOrCalculateHistoricRelativeMeritData()");
switch (this.RelativeMeritDataSource)
{
case "WattTime":
throw new NotImplementedException("WattTime does not currently offer historic relative merit data querying");
break;
case "CustomInternalCalculation":
var calculatedHistoricRelativeMeritResults = this.CalculateHistoricRelativeMeritDataResults(
regionId,
historicStartDateTime,
historicEndDateTime);
// Insert results in the database
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
foreach (var res in calculatedHistoricRelativeMeritResults)
{
_objectModel.InsertOrUpdateCarbonEmissionsRelativeMeritDataPoints(
regionId,
res.Timestamp,
res.EmissionsRelativeMerit,
res.EmissionsRelativeMerit_Forcast);
}
}
break;
default:
Logger.Information(
$"No known defined method of MineOrCalculateHistoricRelativeMeritData supplied to method. Not mining or calculating Historic Relative Merit Data for this region ({regionWattTimeName})",
"CarbonEmissionsMiner.MineOrCalculateHistoricRelativeMeritData()");
return;
}
}
catch (Exception e)
{
Logger.Error(
$"CarbonEmissionsMiner: MineForecastMarginalCarbonResults(): Exception encountered while emissions Data figures for {regionWattTimeName} between {historicStartDateTime} and {historicEndDateTime}.",
"CarbonEmissionsMiner.MineForecastMarginalCarbonResults()",
null,
e);
}
}
/// <summary>
/// InsertTenDatForecastValuesIntoDatabase
/// </summary>
/// <param name="regionId"></param>
/// <param name="results"></param>
private void InsertTenDatForecastValuesIntoDatabase(int regionId, List <HourlyForecast> results)
{
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
foreach (var res in results)
{
var dateTime = res.observationDateTime;
double tempcelcuis;
double.TryParse(res.temp.metric, out tempcelcuis);
double dewPoint;
double.TryParse(res.dewpoint.metric, out dewPoint);
double windSpeedKmPh;
double.TryParse(res.wspd.metric, out windSpeedKmPh);
double windDirectionDegrees;
double.TryParse(res.wdir.degrees, out windDirectionDegrees);
double windChill;
double.TryParse(res.windchill.metric, out windChill);
double uvIndex;
double.TryParse(res.uvi, out uvIndex);
double snow;
double.TryParse(res.snow.metric, out snow);
double pressure;
double.TryParse(res.mslp.metric, out pressure);
double humidityPercentage;
double.TryParse(res.humidity, out humidityPercentage);
double precipitation;
double.TryParse(res.qpf.metric, out precipitation);
var conditionDescription = res.condition ?? string.Empty;
// Clean up "-9999" values that sometimes come back
if (windSpeedKmPh < 0)
{
windSpeedKmPh = -1;
}
_objectModel.InsertOrUpdateWeatherDataPoints(
regionId,
dateTime,
tempcelcuis,
dewPoint,
windSpeedKmPh,
null,
// No wind gusts in Wunderground forecasts
windDirectionDegrees,
windChill,
null,
// No visibilty in Wunderground forecasts
uvIndex,
precipitation,
snow,
pressure,
humidityPercentage,
conditionDescription,
true);
}
}
}
public IEnumerable <string> Get()
{
try
{
var databaseConnectionString = CloudConfigurationManager.GetSetting("SQLAzureDatabaseEntityFrameworkConnectionString");
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
var result = _objectModel.FindAllEmissionsRegions();
return(result);
}
}
catch (Exception e)
{
throw new Exception("Sorry - an exception occured executing the request");
}
}
public IHttpActionResult Get([FromUri] string id, [FromUri] DateTime?startDateTime = null, [FromUri] DateTime?endDateTime = null,
[FromUri] bool returnOnlyNonNullSystemWideEmissionsDataPoints = false, [FromUri] bool returnOnlyNonNullMarginalEmissionsDataPoints = false)
{
try
{
// Set default start and end datetimes
string emissionsRegionFriendlyName = id.IsNullOrWhiteSpace() ? "US_PJM" : id;
var startDateTimeProcessed = startDateTime ?? DateTime.UtcNow.AddHours(-3);
var endDateTimeProcessed = endDateTime ?? DateTime.UtcNow.AddHours(1);
// Query database
var databaseConnectionString = CloudConfigurationManager.GetSetting("SQLAzureDatabaseEntityFrameworkConnectionString");
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
var regionId = _objectModel.FindEmissionsRegion(emissionsRegionFriendlyName).EmissionsRegionID;
var result = _objectModel.FindCarbonEmissionsDataPoints(regionId, startDateTimeProcessed, endDateTimeProcessed).ToList();
// Remove any relavent results as per search parameters passed in - System Wide and Marginal Emissions
if (returnOnlyNonNullSystemWideEmissionsDataPoints)
{
result.RemoveAll(x => x.SystemWideCO2Intensity_gCO2kWh == null);
}
if (returnOnlyNonNullMarginalEmissionsDataPoints)
{
result.RemoveAll(x => x.MarginalCO2Intensity_gCO2kWh == null);
}
// Convert the final list to objects of the common data type in the SmartEnergyAzureDataTypes NuGet package
var webResult = this.ConvertToCarbonEmissionsWebDataPoints(result);
if (result == null)
{
return(this.NotFound());
}
return(Ok(webResult));
}
}
catch (Exception e)
{
throw new Exception("Sorry - an exception occured executing the request");
}
}
public void TestMineHistoricWeatherValues_ByGPS()
{
// Arrange
var startDateTime = new DateTime(2017, 1, 1); // DateTime.Now.AddDays(-10);
var endDateTime = new DateTime(2017, 1, 2); // var startDateTime = DateTime.Now.AddDays(-1);
var latitude = 58.279231;
var longtitude = 6.892410;
var smartGridRegionName = "Norway_Oye";
var timeZone = "Central European Standard Time";
var wundergroundApiUrl = CloudConfigurationManager.GetSetting("WundergroundApiUrl");
var wundergroundApiKey = CloudConfigurationManager.GetSetting("WundergroundApiKey");
var selfThrottlingMethod = "AzureTableStorageCallRecollection";
var maxNumberOfCallsPerMinute = 5;
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddWeatherRegion(smartGridRegionName, timeZone, latitude, longtitude, null).WeatherRegionID;
}
var wundergroundWeatherInteraction =
new WundergroundWeatherInteraction(
selfThrottlingMethod,
maxNumberOfCallsPerMinute);
WeatherDataMiner weatherDataMiner = new WeatherDataMiner(
wundergroundApiUrl,
wundergroundApiKey,
selfThrottlingMethod,
databaseConnectionString,
maxNumberOfCallsPerMinute,
wundergroundWeatherInteraction);
// Act
weatherDataMiner.MineHistoricWeatherValues(startDateTime, endDateTime, latitude, longtitude, regionId);
// Assert
}
/// <summary>
/// Take a CSV File containing CarbonEmissionsDataPoint values, and add them as CarbonEmissionsDataPoint to the database
/// </summary>
/// <param name="folderContainingCsv"></param>
/// <param name="csvFileName"></param>
/// <param name="regionId">RegionId to which the CarbonEmissionsDataPoint values pertain</param>
/// <param name="dateTimeFormat">DateTime format to be passed to DateTime.ParseExact(). E.g. "MM/dd/yyyy HH:mm:ss"</param>
/// <param name="ignoreParsingErrors">True to ignore errors parsing individual rows. False to rethrow the exceptions.</param>
/// <returns></returns>
public void ImportCarbonResultsToDatabaseFromCsv(
string folderContainingCsv,
string csvFileName,
int regionId,
string dateTimeFormat = "MM/dd/yyyy HH:mm:ss",
bool ignoreParsingErrors = true)
{
try
{
Logger.Information(
$"Importing Historic Carbon Results for RegionId {regionId} from CSV {csvFileName}.",
"CarbonEmissionsMiner.ImportCarbonResultsToDatabaseFromCsv()");
var results =
ImportCarbonResultsFromCsv(folderContainingCsv, csvFileName, regionId, dateTimeFormat, ignoreParsingErrors);
using (var objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
foreach (var result in results)
{
// Insert the value into the database
objectModel.InsertOrUpdateCarbonEmissionsDataPoints(
regionId,
result.DateTimeUTC,
result.SystemWideCO2Intensity_gCO2kWh,
null,
result.MarginalCO2Intensity_gCO2kWh,
null);
}
}
}
catch (Exception e)
{
Logger.Error(
$"CarbonEmissionsMiner: ImportCarbonResultsToDatabaseFromCsv(): Exception encountered importing emissions Data from CSV with name {csvFileName} in folder {folderContainingCsv} and inserting them to the database.",
"CarbonEmissionsMiner.ImportCarbonResultsToDatabaseFromCsv()",
null,
e);
}
}
public void TestMineForecastcWeatherValues_ByGPS()
{
// Arrange
var startDateTime = DateTime.UtcNow;
var endDateTime = DateTime.UtcNow.AddDays(10);
var latitude = 53.3498;
var longtitude = -6.2603;
var smartGridRegionName = "Ireland";
var timeZone = "GMT Standard Time";
var darkSkyApiUrl = CloudConfigurationManager.GetSetting("DarkSkyApiUrl");
var darkSkyApiKey = CloudConfigurationManager.GetSetting("DarkSkyApiKey");
var selfThrottlingMethod = "AzureTableStorageCallRecollection";
var maxNumberOfCallsPerMinute = 5;
var maxNumberOfCallsPerDay = 500;
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddWeatherRegion(smartGridRegionName, timeZone, latitude, longtitude, null).WeatherRegionID;
}
var darkSkyWeatherInteraction =
new DarkSkyWeatherInteraction(
selfThrottlingMethod,
maxNumberOfCallsPerMinute,
maxNumberOfCallsPerDay);
DarkSkyWeatherDataMiner weatherDataMiner = new DarkSkyWeatherDataMiner(
darkSkyApiUrl,
darkSkyApiKey,
selfThrottlingMethod,
databaseConnectionString,
maxNumberOfCallsPerMinute,
maxNumberOfCallsPerDay,
darkSkyWeatherInteraction);
// Act
weatherDataMiner.MineForecastWeatherValues(startDateTime, endDateTime, latitude, longtitude, regionId);
// Assert
//Verify that each data point has been recorded in the database
var results = darkSkyWeatherInteraction.GetForecastWeatherData(
darkSkyApiUrl,
darkSkyApiKey,
latitude,
longtitude,
startDateTime,
endDateTime);
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
foreach (var result in results)
{
var dataPoint = _objectModel.FindWeatherDataPoint(regionId, result.dateTime);
Assert.IsNotNull(dataPoint);
}
}
}
/// <summary>
/// Mine historic weather records from the Wunderground Weather service for the given regionSubUrl (e.g. CA/San_Francisco) and add it to the database
/// </summary>
/// <param name="startDateTime">startDateTime</param>
/// <param name="endDateTime">endDateTime</param>
/// <param name="regionSubUrl">Sub Url of the region on the Wunderground weather API e.g. CA/San_Francisco</param>
/// <param name="regionId">regionId of this region in the application's database</param>
public void MineHistoricWeatherValues(
DateTime?startDateTime,
DateTime?endDateTime,
string regionSubUrl,
int regionId)
{
var historicStartDateTime = startDateTime ?? DateTime.Now.AddDays(-2);
var historicEndDateTime = endDateTime ?? DateTime.Now.AddMinutes(15);
try
{
var results = this.wundergroundWeatherInteraction.GetHistoricWeatherData(
this.wundergroundApiUrl,
regionSubUrl,
this.wundergroundApiKey,
historicStartDateTime,
historicEndDateTime);
Logger.Information(
$"Received {results.Count} HistoricWeatherValues Results for region with SubUrl {regionSubUrl} from Wunderground. Inserting them into the database",
"WeatherDataMiner.MineHistoricWeatherValues()");
// Insert results in the database
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
foreach (var res in results)
{
var dateTime = res.observationDateTime;
double tempcelcuis;
double.TryParse(res.tempm, out tempcelcuis);
double dewPoint;
double.TryParse(res.dewptm, out dewPoint);
double humidityPercentage;
double.TryParse(res.hum, out humidityPercentage);
double windSpeedKmPh;
double.TryParse(res.wspdm, out windSpeedKmPh);
double windGustKmPh;
double.TryParse(res.wgustm, out windGustKmPh);
double windDirectionDegrees;
double.TryParse(res.wdird, out windDirectionDegrees);
double windChill;
double.TryParse(res.windchillm, out windChill);
double visibilityMetric;
double.TryParse(res.vism, out visibilityMetric);
double snow;
double.TryParse(res.snow, out snow);
double pressure;
double.TryParse(res.pressurem, out pressure);
double precipitation;
double.TryParse(res.precipm, out precipitation);
var conditionDescription = res.conds;
_objectModel.InsertOrUpdateWeatherDataPoints(
regionId,
dateTime,
tempcelcuis,
dewPoint,
windSpeedKmPh,
windGustKmPh,
windDirectionDegrees,
windChill,
visibilityMetric,
null,
// No UNIndex in Wunderground historical data entries
precipitation,
snow,
pressure,
humidityPercentage,
conditionDescription,
false);
}
}
}
catch (Exception e)
{
Logger.Error(
$"WeatherDataMiner: MineHistoricWeatherValues(): Exception encountered while retrieving historical Weather Data figures for {regionSubUrl} between {historicStartDateTime} and {historicEndDateTime}.",
"WeatherDataMiner.MineHistoricWeatherValues()",
null,
e);
}
}
/// <summary>
/// Mine Or Calculate Relative Merit Data for emissions values between the given dates and store them in the database.
/// The Relative Merit is a value between 0 (meaning best) and 1 (being worst).
/// </summary>
/// <param name="startDateTime">Optional StartDateTime. If not supplied, a default value will be used</param>
/// <param name="endDateTime">Optional endDateTime. If not supplied, a default value will be used</param>
/// <param name="regionWattTimeName">Abbreviation for the required region (e.g. "PJM"). See https://api.watttime.org/faq/#where </param>
/// <param name="regionId">regionId of this region in the application's database</param>
public void MineOrCalculateCarbonEmissionsRelativeMerit(
string regionWattTimeName,
int regionId)
{
var historicStartDateTime = DateTime.UtcNow.AddMinutes(-1);
var historicEndDateTime = DateTime.UtcNow.AddMinutes(15);
try
{
Logger.Information(
$"Entering method for {regionWattTimeName} from WattTime URL {this.wattTimeApiUrl} with method of relative data retrieval / calculation of {this.RelativeMeritDataSource}.",
"CarbonEmissionsMiner.MineOrCalculateHistoricRelativeMeritData()");
switch (this.RelativeMeritDataSource)
{
case "WattTime":
var result = this.wattTimeEmissionsInteraction.GetCarbonEmissionsRelativeMeritResults(
WattTimeV2ApiUrl,
regionWattTimeName,
WattTimeUsername,
WattTimePassword
);
if (result != null)
{
Logger.Information(
$"Received result for RelativeMeritData for {regionWattTimeName} from WattTime. Inserting into the database",
"CarbonEmissionsMiner.MineOrCalculateRelativeMeritData()");
// Insert results in the database
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
var dateTime = result.validUntil;
double emissionsRelativeMerit = result.percent / 100; // Normalise percentage to a value between 0 and 1
_objectModel.InsertOrUpdateCarbonEmissionsRelativeMeritDataPoints(
regionId,
dateTime,
emissionsRelativeMerit,
null);
}
}
else
{
Logger.Information(
$"No result found when requesting RelativeMeritData for {regionWattTimeName} from WattTime at UTC: {DateTime.UtcNow}.e",
"CarbonEmissionsMiner.MineOrCalculateRelativeMeritData()");
}
break;
case "CustomInternalCalculation":
var calculatedHistoricRelativeMeritResults = this.CalculateHistoricRelativeMeritDataResults(
regionId,
historicStartDateTime,
historicEndDateTime);
// Insert results in the database
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
foreach (var res in calculatedHistoricRelativeMeritResults)
{
_objectModel.InsertOrUpdateCarbonEmissionsRelativeMeritDataPoints(
regionId,
res.Timestamp,
res.EmissionsRelativeMerit,
res.EmissionsRelativeMerit_Forcast);
}
}
break;
default:
Logger.Information(
$"No known defined method of MineOrCalculateRelativeMeritData supplied to method. Not mining or calculating Historic Relative Merit Data for this region ({regionWattTimeName})",
"CarbonEmissionsMiner.MineOrCalculateRelativeMeritData()");
return;
}
}
catch (Exception e)
{
Logger.Error(
$"CarbonEmissionsMiner: MineOrCalculateRelativeMeritData(): Exception encountered while emissions Data figures for {regionWattTimeName} between {historicStartDateTime} and {historicEndDateTime}.",
"CarbonEmissionsMiner.MineOrCalculateRelativeMeritData()",
null,
e);
}
}
/// <summary>
/// Take a config element containing the data for a region to be mined, mine the data and save it in the database
/// </summary>
/// <param name="regionConfiguration"></param>
/// <param name="wattTimeApiKeyOverride">Optional wattTimeApiKey to Override what's in the MinerConfigXML File</param>
/// <param name="wundergroundApiKeyOverride">Optional WundergroundApiKey to Override what's in the MinerConfigXML File</param>
public void MineRegionData(ApiMinerConfigLayoutRegion regionConfiguration, string wattTimeApiKeyOverride = null,
string wundergroundApiKeyOverride = null)
{
var regionGroupingName = regionConfiguration.friendlyName;
using (new TimedOperation(
$"Beginning Mining of all data for Region {regionGroupingName}",
"ApiDataMiner.MineRegionData()"))
{
// Mine the regions emissions if an emissions node was supplied
int?emissionsRegionId = null;
if (regionConfiguration.EmissionsMiningRegion != null)
{
var friendlyName = regionConfiguration.EmissionsMiningRegion.friendlyName;
using (
new TimedOperation(
$"Beginning Mining of emissions data for Region {friendlyName}",
"ApiDataMiner.MineRegionData()"))
{
var timeZone = regionConfiguration.EmissionsMiningRegion.TimeZone;
var regionLat = regionConfiguration.EmissionsMiningRegion.Latitude;
var regionLong = regionConfiguration.EmissionsMiningRegion.Longitude;
var regionWattTimeName =
regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation;
var wattTimeApiUrl = regionConfiguration.EmissionsMiningRegion.ApiUrl;
string wattTimeApiKey = null;
if (string.IsNullOrEmpty(wattTimeApiKeyOverride) || wattTimeApiKeyOverride.Equals("none"))
{
wattTimeApiKey = regionConfiguration.EmissionsMiningRegion.ApiKey;
}
else
{
wattTimeApiKey = wattTimeApiKeyOverride;
}
var selfThrottlingMethod = regionConfiguration.EmissionsMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.EmissionsMiningRegion.MaxNumberOfCallsPerMinute;
var historicStartDateTime = DateTime.UtcNow.AddDays(-15);
var historicEndDateTime = DateTime.UtcNow.AddDays(1);
var forecastStartDateTime = DateTime.UtcNow.AddDays(-2);
var forecastEndDateTime = DateTime.UtcNow.AddDays(10);
if (!string.IsNullOrEmpty(wattTimeApiKey) && !wattTimeApiKey.Equals("none"))
{
Logger.Information(
$"About to add Emissions Region and Mine Carbon Emissions Data for {regionWattTimeName} from WattTime URL {wattTimeApiUrl} from {historicStartDateTime} to {historicEndDateTime} for historic data and insert them into the database",
"ApiDataMiner.MineRegionData()");
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
emissionsRegionId =
_objectModel.AddEmissionsRegion(friendlyName, timeZone, regionLat, regionLong,
regionWattTimeName)
.EmissionsRegionID;
CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
selfThrottlingMethod,
this.DatabaseConnectionString,
maxNumberOfCallsPerMinute);
// Mine Recent Actual Data
carbonEmissionsMiner.MineHistoricCarbonResults(
historicStartDateTime,
historicEndDateTime,
regionWattTimeName,
(int)emissionsRegionId);
// Mine Forecast Data
carbonEmissionsMiner.MineForecastMarginalCarbonResults(
forecastStartDateTime,
forecastEndDateTime,
regionWattTimeName,
(int)emissionsRegionId);
}
}
else
{
Logger.Information(
$"No WattTime Api Key was specified. Skipping this region for Emissions.",
"RunAsync()");
}
}
}
// Mine the regions weather if a weather node was supplied
int?weatherRegionId = null;
if (regionConfiguration.WeatherMiningRegion != null)
{
var friendlyName = regionConfiguration.WeatherMiningRegion.friendlyName;
using (
new TimedOperation(
$"Beginning Mining of weather data for Region {friendlyName}",
"ApiDataMiner.MineRegionData()"))
{
var timeZone = regionConfiguration.WeatherMiningRegion.TimeZone;
var regionLat = regionConfiguration.WeatherMiningRegion.Latitude;
var regionLong = regionConfiguration.WeatherMiningRegion.Longitude;
var weatherRegionWundergroundSubUrl =
regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl;
var wundergroundApiUrl = regionConfiguration.WeatherMiningRegion.ApiUrl;
string wundergroundApiKey = null;
if (string.IsNullOrEmpty(wundergroundApiKeyOverride) || wundergroundApiKeyOverride.Equals("none"))
{
wundergroundApiKey = regionConfiguration.WeatherMiningRegion.ApiKey;
}
else
{
wundergroundApiKey = wundergroundApiKeyOverride;
}
var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;
var historicStartDateTime = DateTime.UtcNow.AddDays(-1);
var historicEndDateTime = DateTime.UtcNow.AddDays(1);
if (!string.IsNullOrEmpty(wundergroundApiKey) && !wundergroundApiKey.Equals("none"))
{
Logger.Information(
$"About to add Emissions Region and Mine Carbon Emissions Data for {friendlyName} from Wunderground URL {weatherRegionWundergroundSubUrl} from {historicStartDateTime} to {historicEndDateTime} for historic data and insert them into the database",
"ApiDataMiner.MineRegionData()");
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
weatherRegionId =
_objectModel.AddWeatherRegion(
friendlyName,
timeZone,
regionLat,
regionLong,
weatherRegionWundergroundSubUrl).WeatherRegionID;
WeatherDataMiner weatherDataMiner = new WeatherDataMiner(
wundergroundApiUrl,
wundergroundApiKey,
selfThrottlingMethod,
this.DatabaseConnectionString,
maxNumberOfCallsPerMinute);
switch (regionConfiguration.WeatherMiningRegion.MiningMethod)
{
case "GPS":
// Mine Recent Actual Data
weatherDataMiner.MineHistoricWeatherValues(
historicStartDateTime,
historicEndDateTime,
regionLat,
regionLong,
(int)weatherRegionId);
// Mine Forecast Data
weatherDataMiner.MineTenDayHourlyForecastWeatherValues(
regionLat,
regionLong,
(int)weatherRegionId);
break;
case "WundergroundPageSubUrl":
default:
// Mine Recent Actual Data
weatherDataMiner.MineHistoricWeatherValues(
historicStartDateTime,
historicEndDateTime,
weatherRegionWundergroundSubUrl,
(int)weatherRegionId);
// Mine Forecast Data
weatherDataMiner.MineTenDayHourlyForecastWeatherValues(
weatherRegionWundergroundSubUrl,
(int)weatherRegionId);
break;
}
}
}
else
{
Logger.Information(
$"No Wunderground Api Key was specified. Skipping this region for Weather.",
"RunAsync()");
}
}
}
// Group the Emissions and Weather regions if they haven't already been grouped
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
_objectModel.AddMarketWeatherEmissionsRegionMapping(
regionGroupingName,
null,
weatherRegionId,
emissionsRegionId);
}
}
}
private void InsertHistoricWeatherValuesIntoDatabase(int regionId, List <Observation> results)
{
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
foreach (var res in results)
{
var dateTime = res.observationDateTime;
double tempcelcuis;
double.TryParse(res.tempm, out tempcelcuis);
double dewPoint;
double.TryParse(res.dewptm, out dewPoint);
double humidityPercentage;
double.TryParse(res.hum, out humidityPercentage);
double windSpeedKmPh;
double.TryParse(res.wspdm, out windSpeedKmPh);
double windGustKmPh;
double.TryParse(res.wgustm, out windGustKmPh);
double windDirectionDegrees;
double.TryParse(res.wdird, out windDirectionDegrees);
double windChill;
double.TryParse(res.windchillm, out windChill);
double visibilityMetric;
double.TryParse(res.vism, out visibilityMetric);
double snow;
double.TryParse(res.snow, out snow);
double pressure;
double.TryParse(res.pressurem, out pressure);
double precipitation;
double.TryParse(res.precipm, out precipitation);
var conditionDescription = res.conds ?? string.Empty;
// Clean up "-9999" values that sometimes come back
if (windSpeedKmPh < 0)
{
windSpeedKmPh = -1;
}
if (windGustKmPh < 0)
{
windGustKmPh = -1;
}
_objectModel.InsertOrUpdateWeatherDataPoints(
regionId,
dateTime,
tempcelcuis,
dewPoint,
windSpeedKmPh,
windGustKmPh,
windDirectionDegrees,
windChill,
visibilityMetric,
null,
// No UNIndex in Wunderground historical data entries
precipitation,
snow,
pressure,
humidityPercentage,
conditionDescription,
false);
}
}
}
public void Initialize()
{
this.objectModel = new SmartEnergyOM(this.databaseConnectionString);
}
public void TestMineHistoricSystemWideCarbonResults()
{
// Arrange
var startDateTime = DateTime.Now.AddDays(-10);
var endDateTime = DateTime.Now.AddDays(-1);
var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl");
var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey");
string wattTimeApiV2Url = CloudConfigurationManager.GetSetting("WattTimeApiV2Url");
string WattTimeUsername = CloudConfigurationManager.GetSetting("WattTimeUsername");
string WattTimePassword = CloudConfigurationManager.GetSetting("WattTimePassword");
string WattTimeEmail = CloudConfigurationManager.GetSetting("WattTimeEmail");
string WattTimeOrganization = CloudConfigurationManager.GetSetting("WattTimeOrganization");
var selfThrottlingMethod = "AzureTableStorageCallRecollection";
var maxNumberOfCallsPerMinute = 9;
List <WattTimeBalancingAuthorityInformation> regionsToMine = new List <WattTimeBalancingAuthorityInformation>();
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("ERCOT", "US_ERCOT", "Central Standard Time", 32.79878236662912, -96.77856445062508));
foreach (var region in regionsToMine)
{
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddEmissionsRegion(region.smartGridRegionName, region.timeZone, region.regionLat, region.regionLong)
.EmissionsRegionID;
_objectModel.AddMarketWeatherEmissionsRegionMapping(
region.smartGridRegionName,
null,
null,
regionId);
}
var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute);
CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
wattTimeApiV2Url,
WattTimeUsername,
WattTimePassword,
WattTimeEmail,
WattTimeOrganization,
selfThrottlingMethod,
databaseConnectionString,
maxNumberOfCallsPerMinute,
wattTimeInteraction);
// Act
carbonEmissionsMiner.MineHistoricSystemWideCarbonResults(
startDateTime,
endDateTime,
region.regionWattTimeName,
regionId);
// Assert
// Verify that each data point has been recorded in the database
var results = wattTimeInteraction.GetGenerationMixAndSystemWideEmissionsResults(
wattTimeApiUrl,
region.regionWattTimeName,
startDateTime,
endDateTime,
null,
wattTimeApiKey);
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
foreach (var result in results)
{
var dataPoint = _objectModel.FindCarbonEmissionsDataPoint(regionId, result.timestamp);
Assert.IsNotNull(dataPoint);
}
}
}
}
public void TestMineForecastMarginalCarbonResults()
{
// Arrange
var regionWattTimeName = "PJM";
var smartGridRegionName = "PJM";
var timeZone = "Eastern Standard Time";
var regionLat = 40.348444276169;
var regionLong = -74.6428556442261;
var startDateTime = DateTime.UtcNow.AddDays(-2);
var endDateTime = DateTime.UtcNow.AddDays(10);
var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl");
var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey");
var selfThrottlingMethod = "AzureTableStorageCallRecollection";
var maxNumberOfCallsPerMinute = 9;
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddEmissionsRegion(smartGridRegionName, timeZone, regionLat, regionLong)
.EmissionsRegionID;
}
var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute);
CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
selfThrottlingMethod,
databaseConnectionString,
maxNumberOfCallsPerMinute,
wattTimeInteraction);
// Act
carbonEmissionsMiner.MineForecastMarginalCarbonResults(
startDateTime,
endDateTime,
regionWattTimeName,
regionId);
// Assert
// Verify that each data point has been recorded in the database
var results = wattTimeInteraction.GetForecastMarginalCarbonResults(
wattTimeApiUrl,
regionWattTimeName,
startDateTime,
endDateTime,
null,
wattTimeApiKey);
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
foreach (var result in results)
{
if (result.marginal_carbon.value != null)
{
var dataPoint = _objectModel.FindCarbonEmissionsDataPoints(regionId, result.timestamp);
Assert.IsNotNull(dataPoint);
Assert.AreEqual(
wattTimeInteraction.ConvertLbsPerMWhTo_GCo2PerkWh((double)result.marginal_carbon.value),
dataPoint.MarginalCO2Intensity_gCO2kWh);
}
}
}
}
/// <summary>
/// Calculate the Relative Merit of Marginal Carbon Emissions Values which are present in the database between the given startDateTime and endDateTime.
/// The Relative Merit is a value between 0 (meaning best) and 1 (being worst).
/// </summary>
/// <param name="regionId"></param>
/// <param name="startDateTime"></param>
/// <param name="endDateTime"></param>
/// <returns>List of MarginalCarbonResult.Result representing the Relative Merit of the corresponding Marginal Carbon Emissions Values</returns>
public List <EmissionsRelativeMeritDatapoint> CalculateHistoricRelativeMeritDataResults(int regionId, DateTime startDateTime, DateTime endDateTime)
{
/*** Implement your custom logic here to calculate relative merit data ***/
/* Here is a sample method provided here which simply calculates it's comparison to a one week rolling average for the data */
var results = new List <EmissionsRelativeMeritDatapoint>();
using (var objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
// Get datapoints on which to calculate relative merit
var carbonResultsOnWhichToCalculateRelativeMetit =
objectModel.FindCarbonEmissionsDataPoints(regionId, startDateTime, endDateTime);
foreach (var datapoint in carbonResultsOnWhichToCalculateRelativeMetit)
{
try
{
// Get the last week of data
var startDateOfOneWeekRollingAverage = datapoint.DateTimeUTC.AddDays(-7);
var endDateOfOneWeekRollingAverage = datapoint.DateTimeUTC.AddHours(1);
var emissionsDataPointsWithinWindow =
objectModel.FindCarbonEmissionsDataPoints(regionId, startDateOfOneWeekRollingAverage,
endDateOfOneWeekRollingAverage).Where(a => a.MarginalCO2Intensity_gCO2kWh != null);
// Calculate where this datapoints falls within the range of the last week
if (emissionsDataPointsWithinWindow.Any())
{
double?relativeMerit = null;
double?relativeMerit_Forecast = null;
if (datapoint.MarginalCO2Intensity_gCO2kWh != null)
{
var maxValue = emissionsDataPointsWithinWindow.Max(a => a.MarginalCO2Intensity_gCO2kWh);
var minValue = emissionsDataPointsWithinWindow.Min(a => a.MarginalCO2Intensity_gCO2kWh);
var range = (double)(maxValue - minValue);
relativeMerit =
(datapoint.MarginalCO2Intensity_gCO2kWh - minValue) / range;
// One specicial check: 0 should be reserved for zero emissions. Set anything above zero to .2
if ((relativeMerit < .2) && (datapoint.MarginalCO2Intensity_gCO2kWh > 0))
{
relativeMerit = .2;
}
}
if (datapoint.MarginalCO2Intensity_Forcast_gCO2kWh != null)
{
var maxValue = emissionsDataPointsWithinWindow.Max(a => a.MarginalCO2Intensity_Forcast_gCO2kWh);
var minValue = emissionsDataPointsWithinWindow.Min(a => a.MarginalCO2Intensity_Forcast_gCO2kWh);
var range = (double)(maxValue - minValue);
relativeMerit_Forecast =
(datapoint.MarginalCO2Intensity_Forcast_gCO2kWh - minValue) / range;
// One specicial check: 0 should be reserved for zero emissions. Set anything above zero to .2
if ((relativeMerit_Forecast < .2) && (datapoint.MarginalCO2Intensity_Forcast_gCO2kWh > 0))
{
relativeMerit_Forecast = .2;
}
}
if ((relativeMerit != null) || (relativeMerit_Forecast != null))
{
var emissionsRelativeMeritDataResult =
new EmissionsRelativeMeritDatapoint
{
EmissionsRegionID = regionId,
Timestamp = datapoint.DateTimeUTC,
EmissionsRelativeMerit = relativeMerit,
EmissionsRelativeMerit_Forcast = relativeMerit_Forecast
};
results.Add(emissionsRelativeMeritDataResult);
}
}
}
catch (Exception e)
{
Logger.Error(
$"CarbonEmissionsMiner: CalculateHistoricRelativeMeritDataResults(): Exception encountered Calculate the Merit of datapoint at {datapoint.DateTimeUTC} for RegionId {datapoint.EmissionsRegionID}.",
"CarbonEmissionsMiner.CalculateHistoricRelativeMeritDataResults()",
null,
e);
}
}
}
return(results);
}
public void TestMineHistoricMarginalCarbonResults()
{
// Arrange
var startDateTime = DateTime.Now.AddDays(-10);
var endDateTime = DateTime.Now.AddDays(-9);
var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl");
var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey");
var selfThrottlingMethod = "AzureTableStorageCallRecollection";
var maxNumberOfCallsPerMinute = 9;
List <WattTimeBalancingAuthorityInformation> regionsToMine =
new List <WattTimeBalancingAuthorityInformation>
{
new WattTimeBalancingAuthorityInformation(
"PJM",
"US_PJM",
"Eastern Standard Time",
40.348444276169,
-74.6428556442261)
};
foreach (var region in regionsToMine)
{
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddEmissionsRegion(region.smartGridRegionName, region.timeZone, region.regionLat, region.regionLong)
.EmissionsRegionID;
_objectModel.AddMarketWeatherEmissionsRegionMapping(
region.smartGridRegionName,
null,
null,
regionId);
}
var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute);
CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
selfThrottlingMethod,
databaseConnectionString,
maxNumberOfCallsPerMinute,
wattTimeInteraction);
// Act
carbonEmissionsMiner.MineHistoricMarginalCarbonResults(
startDateTime,
endDateTime,
region.regionWattTimeName,
regionId);
// Assert
// Verify that each data point has been recorded in the database
var results = wattTimeInteraction.GetObservedMarginalCarbonResults(
wattTimeApiUrl,
region.regionWattTimeName,
startDateTime,
endDateTime,
null,
wattTimeApiKey);
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
foreach (var result in results)
{
var dataPoint = _objectModel.FindCarbonEmissionsDataPoints(regionId, result.timestamp);
Assert.IsNotNull(dataPoint);
}
}
}
}
public void TestMineHistoricMarginalCarbonResults()
{
// Arrange
var startDateTime = DateTime.Now.AddDays(-10);
var endDateTime = DateTime.Now.AddDays(-1);
var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl");
var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey");
string wattTimeApiV2Url = CloudConfigurationManager.GetSetting("WattTimeApiV2Url");
string WattTimeUsername = CloudConfigurationManager.GetSetting("WattTimeUsername");
string WattTimePassword = CloudConfigurationManager.GetSetting("WattTimePassword");
string WattTimeEmail = CloudConfigurationManager.GetSetting("WattTimeEmail");
string WattTimeOrganization = CloudConfigurationManager.GetSetting("WattTimeOrganization");
var selfThrottlingMethod = "AzureTableStorageCallRecollection";
var maxNumberOfCallsPerMinute = 9;
List <WattTimeBalancingAuthorityInformation> regionsToMine = new List <WattTimeBalancingAuthorityInformation>();
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PSEI", "US_PugetSoundEnergy", "Pacific Standard Time", 47.68009593341535, -122.11638450372567));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("BPA", "US_BPA", "Pacific Standard Time", 40.348444276169, -74.6428556442261));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("CAISO", "US_CAISO", "Pacific Standard Time", 41.7324, -123.409423));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("ERCOT", "US_ERCOT", "Central Standard Time", 32.79878236662912, -96.77856445062508));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("ISONE", "US_ISONewEngland", "Eastern Standard Time", 42.70864591994315, -72.16918945062508));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("MISO", "US_UpperMidwestISO", "Central Standard Time", 41.91853269857261, -93.55193137872567));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("SPP", "US_SouthwesternPublicServiceISO", "Eastern Standard Time", 34.41133502036136, -103.19243430841317));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PJM", "US_PJM", "Eastern Standard Time", 40.348444276169, -74.6428556442261));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PJM_ATLANTIC", "US_PJM_ATLANTIC", "Eastern Standard Time", 40.566564, -76.98465597395705));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PJM_SOUTH", "US_PJM_SOUTH", "Eastern Standard Time", 37.44276433719146, -76.87479269270705));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PJM_WEST", "US_PJM_WEST", "Eastern Standard Time", 38.69484915602888, -85.11453878645705));
foreach (var region in regionsToMine)
{
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddEmissionsRegion(region.smartGridRegionName, region.timeZone, region.regionLat, region.regionLong)
.EmissionsRegionID;
_objectModel.AddMarketWeatherEmissionsRegionMapping(
region.smartGridRegionName,
null,
null,
regionId);
}
var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute);
CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
wattTimeApiV2Url,
WattTimeUsername,
WattTimePassword,
WattTimeEmail,
WattTimeOrganization,
selfThrottlingMethod,
databaseConnectionString,
maxNumberOfCallsPerMinute,
wattTimeInteraction);
// Act
carbonEmissionsMiner.MineHistoricMarginalCarbonResults(
startDateTime,
endDateTime,
region.regionWattTimeName,
regionId);
// Assert
// Verify that each data point has been recorded in the database
var results = wattTimeInteraction.GetObservedMarginalCarbonResults(
wattTimeApiV2Url,
region.regionWattTimeName,
WattTimeUsername,
WattTimePassword,
startDateTime,
endDateTime,
null,
null);
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
foreach (var result in results)
{
var dataPoint = _objectModel.FindCarbonEmissionsDataPoint(regionId, result.point_time);
Assert.IsNotNull(dataPoint);
}
}
}
}
/// <summary>
/// Retrieve forecast weather data from the Wunderground Weather service for the given regionSubUrl (e.g. CA/San_Francisco) and add it to the database
/// </summary>
/// <param name="regionSubUrl">Sub Url of the region on the Wunderground weather API e.g. CA/San_Francisco</param>
/// <param name="regionId">regionId of this region in the application's database</param>
public void MineTenDayHourlyForecastWeatherValues(string regionSubUrl, int regionId)
{
try
{
var results =
this.wundergroundWeatherInteraction.GetTenDayHourlyForecastWeatherData(
this.wundergroundApiUrl,
regionSubUrl,
this.wundergroundApiKey);
Logger.Information(
$"Received {results.Count} TenDayHourlyForecastWeatherValues Results for region with SubUrl {regionSubUrl} from Wunderground. Inserting them into the database",
"WeatherDataMiner.MineTenDayHourlyForecastWeatherValues()");
// Insert results in the database
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
foreach (var res in results)
{
var dateTime = res.observationDateTime;
double tempcelcuis;
double.TryParse(res.temp.metric, out tempcelcuis);
double dewPoint;
double.TryParse(res.dewpoint.metric, out dewPoint);
double windSpeedKmPh;
double.TryParse(res.wspd.metric, out windSpeedKmPh);
double windDirectionDegrees;
double.TryParse(res.wdir.degrees, out windDirectionDegrees);
double windChill;
double.TryParse(res.windchill.metric, out windChill);
double uvIndex;
double.TryParse(res.uvi, out uvIndex);
double snow;
double.TryParse(res.snow.metric, out snow);
double pressure;
double.TryParse(res.mslp.metric, out pressure);
double humidityPercentage;
double.TryParse(res.humidity, out humidityPercentage);
double precipitation;
double.TryParse(res.qpf.metric, out precipitation);
var conditionDescription = res.condition;
_objectModel.InsertOrUpdateWeatherDataPoints(
regionId,
dateTime,
tempcelcuis,
dewPoint,
windSpeedKmPh,
null,
// No wind gusts in Wunderground forecasts
windDirectionDegrees,
windChill,
null,
// No visibilty in Wunderground forecasts
uvIndex,
precipitation,
snow,
pressure,
humidityPercentage,
conditionDescription,
true);
}
}
}
catch (Exception e)
{
Logger.Error(
$"WeatherDataMiner: MineHistoricWeatherValues(): Exception encountered while retrieving historical Weather Data figures for {regionSubUrl}.",
"WeatherDataMiner.MineTenDayHourlyForecastWeatherValues()",
null,
e);
}
}
public void TestParseMinerSettingsFileAndMineData()
{
// Arrange
var ConfigPath = @".\ApiDataMinerConfigs\ApiDataMinerConfigs.xml";
// Act
var apiDataMiner = new ApiDataMiner(databaseConnectionString);
apiDataMiner.ParseMinerSettingsFileAndMineData(ConfigPath);
// Assert
using (var streamReader = new StreamReader(ConfigPath))
{
var xmlSerializer = new XmlSerializer(typeof(ApiMinerConfigLayout));
var minerConfigs = (ApiMinerConfigLayout)xmlSerializer.Deserialize(streamReader);
foreach (var regionConfiguration in minerConfigs.Regions)
{
// Verify emissions were mined successfully for each region in the Config File
if (regionConfiguration.EmissionsMiningRegion != null)
{
var emissionsRegionName = regionConfiguration.EmissionsMiningRegion.friendlyName;
var timeZone = regionConfiguration.EmissionsMiningRegion.TimeZone;
var regionLat = regionConfiguration.EmissionsMiningRegion.Latitude;
var regionLong = regionConfiguration.EmissionsMiningRegion.Longitude;
var regionWattTimeName = regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation;
var wattTimeApiUrl = regionConfiguration.EmissionsMiningRegion.ApiUrl;
var wattTimeApiKey = regionConfiguration.EmissionsMiningRegion.ApiKey;
var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;
var startDateTime = DateTime.UtcNow.AddDays(-2);
var endDateTime = DateTime.UtcNow.AddDays(10);
var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute);
var results = wattTimeInteraction.GetObservedMarginalCarbonResults(
wattTimeApiUrl,
regionWattTimeName,
startDateTime,
endDateTime,
null,
wattTimeApiKey);
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddEmissionsRegion(emissionsRegionName, timeZone, regionLat, regionLong)
.EmissionsRegionID;
}
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
foreach (var result in results)
{
var dataPoint = _objectModel.FindCarbonEmissionsDataPoints(regionId, result.timestamp);
Assert.IsNotNull(dataPoint);
}
}
}
// Verify weather was mined successfully for each region in the Config File
if (regionConfiguration.WeatherMiningRegion != null)
{
var emissionsRegionName = regionConfiguration.WeatherMiningRegion.friendlyName;
var timeZone = regionConfiguration.WeatherMiningRegion.TimeZone;
var regionLat = regionConfiguration.WeatherMiningRegion.Latitude;
var regionLong = regionConfiguration.WeatherMiningRegion.Longitude;
var weatherRegionWundergroundSubUrl =
regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl;
var wundergroundApiUrl = regionConfiguration.WeatherMiningRegion.ApiUrl;
var wundergroundApiKey = regionConfiguration.WeatherMiningRegion.ApiKey;
var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;
var wundergroundWeatherInteraction = new WundergroundWeatherInteraction(
selfThrottlingMethod,
maxNumberOfCallsPerMinute);
var results =
wundergroundWeatherInteraction.GetTenDayHourlyForecastWeatherData(
wundergroundApiUrl,
weatherRegionWundergroundSubUrl,
wundergroundApiKey);
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddWeatherRegion(emissionsRegionName, timeZone, regionLat, regionLong)
.WeatherRegionID;
}
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
foreach (var result in results)
{
var dataPoint = _objectModel.FindWeatherDataPoints(
regionId,
result.observationDateTime);
Assert.IsNotNull(dataPoint);
}
}
}
}
}
}
public void TestCalculateHistoricRelativeMeritDataResults()
{
// Arrange
var smartGridRegionName = "US_PJM";
var startDateTime = DateTime.UtcNow.AddDays(-10);
var endDateTime = DateTime.UtcNow.AddDays(-9);
var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl");
var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey");
string wattTimeApiV2Url = CloudConfigurationManager.GetSetting("WattTimeApiV2Url");
string WattTimeUsername = CloudConfigurationManager.GetSetting("WattTimeUsername");
string WattTimePassword = CloudConfigurationManager.GetSetting("WattTimePassword");
string WattTimeEmail = CloudConfigurationManager.GetSetting("WattTimeEmail");
string WattTimeOrganization = CloudConfigurationManager.GetSetting("WattTimeOrganization");
var selfThrottlingMethod = "AzureTableStorageCallRecollection";
var maxNumberOfCallsPerMinute = 200;
var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute);
var carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
wattTimeApiV2Url,
WattTimeUsername,
WattTimePassword,
WattTimeEmail,
WattTimeOrganization,
selfThrottlingMethod,
databaseConnectionString,
maxNumberOfCallsPerMinute,
wattTimeInteraction);
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId = _objectModel.FindEmissionsRegion(smartGridRegionName).EmissionsRegionID;
}
Assert.IsNotNull(regionId, "Couldn't find specified emissions region in the database");
// Act
var results = carbonEmissionsMiner.CalculateHistoricRelativeMeritDataResults(
regionId,
startDateTime,
endDateTime
);
// Assert
foreach (var item in results)
{
if (item.EmissionsRelativeMerit != null)
{
Assert.IsTrue(item.EmissionsRelativeMerit >= 0, "item.EmissionsRelativeMerit >= 0");
Assert.IsTrue(item.EmissionsRelativeMerit <= 1, "item.EmissionsRelativeMerit <= 1");
}
if (item.EmissionsRelativeMerit_Forcast != null)
{
Assert.IsTrue(item.EmissionsRelativeMerit_Forcast >= 0, "item.EmissionsRelativeMerit_Forcast >= 0");
Assert.IsTrue(item.EmissionsRelativeMerit_Forcast <= 1, "item.EmissionsRelativeMerit_Forcast <= 1");
}
}
}
/// <summary>
/// Take a config element containing the data for a region to be mined, mine the data and save it in the database
/// </summary>
/// <param name="regionConfiguration"></param>
public void MineRegionData(ApiMinerConfigLayoutRegion regionConfiguration)
{
var regionGroupingName = regionConfiguration.friendlyName;
using (new TimedOperation(
$"Beginning Mining of all data for Region {regionGroupingName}",
"ApiDataMiner.MineRegionData()"))
{
// Mine the regions emissions if an emissions node was supplied
int?emissionsRegionId = null;
if (regionConfiguration.EmissionsMiningRegion != null)
{
var friendlyName = regionConfiguration.EmissionsMiningRegion.friendlyName;
using (
new TimedOperation(
$"Beginning Mining of emissions data for Region {friendlyName}",
"ApiDataMiner.MineRegionData()"))
{
var timeZone = regionConfiguration.EmissionsMiningRegion.TimeZone;
var regionLat = regionConfiguration.EmissionsMiningRegion.Latitude;
var regionLong = regionConfiguration.EmissionsMiningRegion.Longitude;
var regionWattTimeName = regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation;
var wattTimeApiUrl = regionConfiguration.EmissionsMiningRegion.ApiUrl;
var wattTimeApiKey = regionConfiguration.EmissionsMiningRegion.ApiKey;
var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;
var historicStartDateTime = DateTime.UtcNow.AddDays(-15);
var historicEndDateTime = DateTime.UtcNow.AddDays(1);
var forecastStartDateTime = DateTime.UtcNow.AddDays(-2);
var forecastEndDateTime = DateTime.UtcNow.AddDays(10);
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
emissionsRegionId =
_objectModel.AddEmissionsRegion(friendlyName, timeZone, regionLat, regionLong)
.EmissionsRegionID;
CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
selfThrottlingMethod,
this.DatabaseConnectionString,
maxNumberOfCallsPerMinute);
// Mine Recent Actual Data
carbonEmissionsMiner.MineHistoricCarbonResults(
historicStartDateTime,
historicEndDateTime,
regionWattTimeName,
(int)emissionsRegionId);
// Mine Forecast Data
carbonEmissionsMiner.MineForecastMarginalCarbonResults(
forecastStartDateTime,
forecastEndDateTime,
regionWattTimeName,
(int)emissionsRegionId);
}
}
}
// Mine the regions emissions if an emissions node was supplied
int?weatherRegionId = null;
if (regionConfiguration.WeatherMiningRegion != null)
{
var friendlyName = regionConfiguration.WeatherMiningRegion.friendlyName;
using (
new TimedOperation(
$"Beginning Mining of weather data for Region {friendlyName}",
"ApiDataMiner.MineRegionData()"))
{
var timeZone = regionConfiguration.WeatherMiningRegion.TimeZone;
var regionLat = regionConfiguration.WeatherMiningRegion.Latitude;
var regionLong = regionConfiguration.WeatherMiningRegion.Longitude;
var weatherRegionWundergroundSubUrl =
regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl;
var wundergroundApiUrl = regionConfiguration.WeatherMiningRegion.ApiUrl;
var wundergroundApiKey = regionConfiguration.WeatherMiningRegion.ApiKey;
var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;
var historicStartDateTime = DateTime.UtcNow.AddDays(-1);
var historicEndDateTime = DateTime.UtcNow.AddDays(1);
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
weatherRegionId =
_objectModel.AddWeatherRegion(
friendlyName,
timeZone,
regionLat,
regionLong,
weatherRegionWundergroundSubUrl).WeatherRegionID;
WeatherDataMiner weatherDataMiner = new WeatherDataMiner(
wundergroundApiUrl,
wundergroundApiKey,
selfThrottlingMethod,
this.DatabaseConnectionString,
maxNumberOfCallsPerMinute);
// Mine Recent Actual Data
weatherDataMiner.MineHistoricWeatherValues(
historicStartDateTime,
historicEndDateTime,
weatherRegionWundergroundSubUrl,
(int)weatherRegionId);
// Mine Forecast Data
weatherDataMiner.MineTenDayHourlyForecastWeatherValues(
weatherRegionWundergroundSubUrl,
(int)weatherRegionId);
}
}
}
// Group the Emissions and Weather regions if they haven't already been grouped
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
_objectModel.AddMarketWeatherEmissionsRegionMapping(
regionGroupingName,
null,
weatherRegionId,
emissionsRegionId);
}
}
}
public void TestMineOrCalculateCarbonEmissionsRelativeMerit_CustomInternalCalculation()
{
// Arrange
var regionWattTimeName = "PJM";
var smartGridRegionName = "US_PJM";
var startDateTime = DateTime.UtcNow.AddDays(-10);
var endDateTime = DateTime.UtcNow.AddDays(-9);
var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl");
var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey");
string wattTimeApiV2Url = CloudConfigurationManager.GetSetting("WattTimeApiV2Url");
string WattTimeUsername = CloudConfigurationManager.GetSetting("WattTimeUsername");
string WattTimePassword = CloudConfigurationManager.GetSetting("WattTimePassword");
string WattTimeEmail = CloudConfigurationManager.GetSetting("WattTimeEmail");
string WattTimeOrganization = CloudConfigurationManager.GetSetting("WattTimeOrganization");
var selfThrottlingMethod = "AzureTableStorageCallRecollection";
var maxNumberOfCallsPerMinute = 200;
var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute);
CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
wattTimeApiV2Url,
WattTimeUsername,
WattTimePassword,
WattTimeEmail,
WattTimeOrganization,
selfThrottlingMethod,
databaseConnectionString,
maxNumberOfCallsPerMinute,
wattTimeInteraction,
"CustomInternalCalculation");
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId = _objectModel.FindEmissionsRegion(smartGridRegionName).EmissionsRegionID;
}
Assert.IsNotNull(regionId, "Couldn't find specified emissions region in the database");
// Act
carbonEmissionsMiner.MineOrCalculateCarbonEmissionsRelativeMerit(
regionWattTimeName,
regionId);
// Assert
// Verify that each data point has been recorded in the database
var results = carbonEmissionsMiner.CalculateHistoricRelativeMeritDataResults(
regionId,
startDateTime,
endDateTime);
foreach (var result in results)
{
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
var dataPoint = _objectModel.FindCarbonEmissionsRelativeMeritDataPoints(regionId, result.Timestamp.AddMinutes(-5), result.Timestamp.AddMinutes(15));
Assert.IsNotNull(dataPoint);
}
}
}
