/// <summary>
/// Take a config element containing the data for a region to be mined, mine the data and save it in the database
/// </summary>
/// <param name="regionConfiguration"></param>
public void MineRegionData(ApiMinerConfigLayoutRegion regionConfiguration)
{
var regionGroupingName = regionConfiguration.friendlyName;
using (new TimedOperation(
$"Beginning Mining of all data for Region {regionGroupingName}",
"ApiDataMiner.MineRegionData()"))
{
// Mine the regions emissions if an emissions node was supplied
int?emissionsRegionId = null;
if (regionConfiguration.EmissionsMiningRegion != null)
{
var friendlyName = regionConfiguration.EmissionsMiningRegion.friendlyName;
using (
new TimedOperation(
$"Beginning Mining of emissions data for Region {friendlyName}",
"ApiDataMiner.MineRegionData()"))
{
var timeZone = regionConfiguration.EmissionsMiningRegion.TimeZone;
var regionLat = regionConfiguration.EmissionsMiningRegion.Latitude;
var regionLong = regionConfiguration.EmissionsMiningRegion.Longitude;
var regionWattTimeName = regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation;
var wattTimeApiUrl = regionConfiguration.EmissionsMiningRegion.ApiUrl;
var wattTimeApiKey = regionConfiguration.EmissionsMiningRegion.ApiKey;
var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;
var historicStartDateTime = DateTime.UtcNow.AddDays(-15);
var historicEndDateTime = DateTime.UtcNow.AddDays(1);
var forecastStartDateTime = DateTime.UtcNow.AddDays(-2);
var forecastEndDateTime = DateTime.UtcNow.AddDays(10);
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
emissionsRegionId =
_objectModel.AddEmissionsRegion(friendlyName, timeZone, regionLat, regionLong)
.EmissionsRegionID;
CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
selfThrottlingMethod,
this.DatabaseConnectionString,
maxNumberOfCallsPerMinute);
// Mine Recent Actual Data
carbonEmissionsMiner.MineHistoricCarbonResults(
historicStartDateTime,
historicEndDateTime,
regionWattTimeName,
(int)emissionsRegionId);
// Mine Forecast Data
carbonEmissionsMiner.MineForecastMarginalCarbonResults(
forecastStartDateTime,
forecastEndDateTime,
regionWattTimeName,
(int)emissionsRegionId);
}
}
}
// Mine the regions emissions if an emissions node was supplied
int?weatherRegionId = null;
if (regionConfiguration.WeatherMiningRegion != null)
{
var friendlyName = regionConfiguration.WeatherMiningRegion.friendlyName;
using (
new TimedOperation(
$"Beginning Mining of weather data for Region {friendlyName}",
"ApiDataMiner.MineRegionData()"))
{
var timeZone = regionConfiguration.WeatherMiningRegion.TimeZone;
var regionLat = regionConfiguration.WeatherMiningRegion.Latitude;
var regionLong = regionConfiguration.WeatherMiningRegion.Longitude;
var weatherRegionWundergroundSubUrl =
regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl;
var wundergroundApiUrl = regionConfiguration.WeatherMiningRegion.ApiUrl;
var wundergroundApiKey = regionConfiguration.WeatherMiningRegion.ApiKey;
var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;
var historicStartDateTime = DateTime.UtcNow.AddDays(-1);
var historicEndDateTime = DateTime.UtcNow.AddDays(1);
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
weatherRegionId =
_objectModel.AddWeatherRegion(
friendlyName,
timeZone,
regionLat,
regionLong,
weatherRegionWundergroundSubUrl).WeatherRegionID;
WeatherDataMiner weatherDataMiner = new WeatherDataMiner(
wundergroundApiUrl,
wundergroundApiKey,
selfThrottlingMethod,
this.DatabaseConnectionString,
maxNumberOfCallsPerMinute);
// Mine Recent Actual Data
weatherDataMiner.MineHistoricWeatherValues(
historicStartDateTime,
historicEndDateTime,
weatherRegionWundergroundSubUrl,
(int)weatherRegionId);
// Mine Forecast Data
weatherDataMiner.MineTenDayHourlyForecastWeatherValues(
weatherRegionWundergroundSubUrl,
(int)weatherRegionId);
}
}
}
// Group the Emissions and Weather regions if they haven't already been grouped
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
_objectModel.AddMarketWeatherEmissionsRegionMapping(
regionGroupingName,
null,
weatherRegionId,
emissionsRegionId);
}
}
}
public void TestMineHistoricMarginalCarbonResults()
{
// Arrange
var startDateTime = DateTime.Now.AddDays(-10);
var endDateTime = DateTime.Now.AddDays(-9);
var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl");
var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey");
var selfThrottlingMethod = "AzureTableStorageCallRecollection";
var maxNumberOfCallsPerMinute = 9;
List <WattTimeBalancingAuthorityInformation> regionsToMine =
new List <WattTimeBalancingAuthorityInformation>
{
new WattTimeBalancingAuthorityInformation(
"PJM",
"US_PJM",
"Eastern Standard Time",
40.348444276169,
-74.6428556442261)
};
foreach (var region in regionsToMine)
{
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddEmissionsRegion(region.smartGridRegionName, region.timeZone, region.regionLat, region.regionLong)
.EmissionsRegionID;
_objectModel.AddMarketWeatherEmissionsRegionMapping(
region.smartGridRegionName,
null,
null,
regionId);
}
var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute);
CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
selfThrottlingMethod,
databaseConnectionString,
maxNumberOfCallsPerMinute,
wattTimeInteraction);
// Act
carbonEmissionsMiner.MineHistoricMarginalCarbonResults(
startDateTime,
endDateTime,
region.regionWattTimeName,
regionId);
// Assert
// Verify that each data point has been recorded in the database
var results = wattTimeInteraction.GetObservedMarginalCarbonResults(
wattTimeApiUrl,
region.regionWattTimeName,
startDateTime,
endDateTime,
null,
wattTimeApiKey);
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
foreach (var result in results)
{
var dataPoint = _objectModel.FindCarbonEmissionsDataPoints(regionId, result.timestamp);
Assert.IsNotNull(dataPoint);
}
}
}
}
public void TestMineHistoricSystemWideCarbonResults()
{
// Arrange
var startDateTime = DateTime.Now.AddDays(-10);
var endDateTime = DateTime.Now.AddDays(-1);
var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl");
var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey");
string wattTimeApiV2Url = CloudConfigurationManager.GetSetting("WattTimeApiV2Url");
string WattTimeUsername = CloudConfigurationManager.GetSetting("WattTimeUsername");
string WattTimePassword = CloudConfigurationManager.GetSetting("WattTimePassword");
string WattTimeEmail = CloudConfigurationManager.GetSetting("WattTimeEmail");
string WattTimeOrganization = CloudConfigurationManager.GetSetting("WattTimeOrganization");
var selfThrottlingMethod = "AzureTableStorageCallRecollection";
var maxNumberOfCallsPerMinute = 9;
List <WattTimeBalancingAuthorityInformation> regionsToMine = new List <WattTimeBalancingAuthorityInformation>();
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("ERCOT", "US_ERCOT", "Central Standard Time", 32.79878236662912, -96.77856445062508));
foreach (var region in regionsToMine)
{
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddEmissionsRegion(region.smartGridRegionName, region.timeZone, region.regionLat, region.regionLong)
.EmissionsRegionID;
_objectModel.AddMarketWeatherEmissionsRegionMapping(
region.smartGridRegionName,
null,
null,
regionId);
}
var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute);
CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
wattTimeApiV2Url,
WattTimeUsername,
WattTimePassword,
WattTimeEmail,
WattTimeOrganization,
selfThrottlingMethod,
databaseConnectionString,
maxNumberOfCallsPerMinute,
wattTimeInteraction);
// Act
carbonEmissionsMiner.MineHistoricSystemWideCarbonResults(
startDateTime,
endDateTime,
region.regionWattTimeName,
regionId);
// Assert
// Verify that each data point has been recorded in the database
var results = wattTimeInteraction.GetGenerationMixAndSystemWideEmissionsResults(
wattTimeApiUrl,
region.regionWattTimeName,
startDateTime,
endDateTime,
null,
wattTimeApiKey);
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
foreach (var result in results)
{
var dataPoint = _objectModel.FindCarbonEmissionsDataPoint(regionId, result.timestamp);
Assert.IsNotNull(dataPoint);
}
}
}
}
public void TestMineHistoricMarginalCarbonResults()
{
// Arrange
var startDateTime = DateTime.Now.AddDays(-10);
var endDateTime = DateTime.Now.AddDays(-1);
var wattTimeApiUrl = CloudConfigurationManager.GetSetting("WattTimeApiUrl");
var wattTimeApiKey = CloudConfigurationManager.GetSetting("WattTimeApiKey");
string wattTimeApiV2Url = CloudConfigurationManager.GetSetting("WattTimeApiV2Url");
string WattTimeUsername = CloudConfigurationManager.GetSetting("WattTimeUsername");
string WattTimePassword = CloudConfigurationManager.GetSetting("WattTimePassword");
string WattTimeEmail = CloudConfigurationManager.GetSetting("WattTimeEmail");
string WattTimeOrganization = CloudConfigurationManager.GetSetting("WattTimeOrganization");
var selfThrottlingMethod = "AzureTableStorageCallRecollection";
var maxNumberOfCallsPerMinute = 9;
List <WattTimeBalancingAuthorityInformation> regionsToMine = new List <WattTimeBalancingAuthorityInformation>();
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PSEI", "US_PugetSoundEnergy", "Pacific Standard Time", 47.68009593341535, -122.11638450372567));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("BPA", "US_BPA", "Pacific Standard Time", 40.348444276169, -74.6428556442261));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("CAISO", "US_CAISO", "Pacific Standard Time", 41.7324, -123.409423));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("ERCOT", "US_ERCOT", "Central Standard Time", 32.79878236662912, -96.77856445062508));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("ISONE", "US_ISONewEngland", "Eastern Standard Time", 42.70864591994315, -72.16918945062508));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("MISO", "US_UpperMidwestISO", "Central Standard Time", 41.91853269857261, -93.55193137872567));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("SPP", "US_SouthwesternPublicServiceISO", "Eastern Standard Time", 34.41133502036136, -103.19243430841317));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PJM", "US_PJM", "Eastern Standard Time", 40.348444276169, -74.6428556442261));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PJM_ATLANTIC", "US_PJM_ATLANTIC", "Eastern Standard Time", 40.566564, -76.98465597395705));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PJM_SOUTH", "US_PJM_SOUTH", "Eastern Standard Time", 37.44276433719146, -76.87479269270705));
regionsToMine.Add(new WattTimeBalancingAuthorityInformation("PJM_WEST", "US_PJM_WEST", "Eastern Standard Time", 38.69484915602888, -85.11453878645705));
foreach (var region in regionsToMine)
{
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddEmissionsRegion(region.smartGridRegionName, region.timeZone, region.regionLat, region.regionLong)
.EmissionsRegionID;
_objectModel.AddMarketWeatherEmissionsRegionMapping(
region.smartGridRegionName,
null,
null,
regionId);
}
var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute);
CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
wattTimeApiV2Url,
WattTimeUsername,
WattTimePassword,
WattTimeEmail,
WattTimeOrganization,
selfThrottlingMethod,
databaseConnectionString,
maxNumberOfCallsPerMinute,
wattTimeInteraction);
// Act
carbonEmissionsMiner.MineHistoricMarginalCarbonResults(
startDateTime,
endDateTime,
region.regionWattTimeName,
regionId);
// Assert
// Verify that each data point has been recorded in the database
var results = wattTimeInteraction.GetObservedMarginalCarbonResults(
wattTimeApiV2Url,
region.regionWattTimeName,
WattTimeUsername,
WattTimePassword,
startDateTime,
endDateTime,
null,
null);
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
foreach (var result in results)
{
var dataPoint = _objectModel.FindCarbonEmissionsDataPoint(regionId, result.point_time);
Assert.IsNotNull(dataPoint);
}
}
}
}
/// <summary>
/// Take a config element containing the data for a region to be mined, mine the data and save it in the database
/// </summary>
/// <param name="regionConfiguration"></param>
/// <param name="wattTimeApiKeyOverride">Optional wattTimeApiKey to Override what's in the MinerConfigXML File</param>
/// <param name="wundergroundApiKeyOverride">Optional WundergroundApiKey to Override what's in the MinerConfigXML File</param>
public void MineRegionData(ApiMinerConfigLayoutRegion regionConfiguration, string wattTimeApiKeyOverride = null,
string wundergroundApiKeyOverride = null)
{
var regionGroupingName = regionConfiguration.friendlyName;
using (new TimedOperation(
$"Beginning Mining of all data for Region {regionGroupingName}",
"ApiDataMiner.MineRegionData()"))
{
// Mine the regions emissions if an emissions node was supplied
int?emissionsRegionId = null;
if (regionConfiguration.EmissionsMiningRegion != null)
{
var friendlyName = regionConfiguration.EmissionsMiningRegion.friendlyName;
using (
new TimedOperation(
$"Beginning Mining of emissions data for Region {friendlyName}",
"ApiDataMiner.MineRegionData()"))
{
var timeZone = regionConfiguration.EmissionsMiningRegion.TimeZone;
var regionLat = regionConfiguration.EmissionsMiningRegion.Latitude;
var regionLong = regionConfiguration.EmissionsMiningRegion.Longitude;
var regionWattTimeName =
regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation;
var wattTimeApiUrl = regionConfiguration.EmissionsMiningRegion.ApiUrl;
string wattTimeApiKey = null;
if (string.IsNullOrEmpty(wattTimeApiKeyOverride) || wattTimeApiKeyOverride.Equals("none"))
{
wattTimeApiKey = regionConfiguration.EmissionsMiningRegion.ApiKey;
}
else
{
wattTimeApiKey = wattTimeApiKeyOverride;
}
var selfThrottlingMethod = regionConfiguration.EmissionsMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.EmissionsMiningRegion.MaxNumberOfCallsPerMinute;
var historicStartDateTime = DateTime.UtcNow.AddDays(-15);
var historicEndDateTime = DateTime.UtcNow.AddDays(1);
var forecastStartDateTime = DateTime.UtcNow.AddDays(-2);
var forecastEndDateTime = DateTime.UtcNow.AddDays(10);
if (!string.IsNullOrEmpty(wattTimeApiKey) && !wattTimeApiKey.Equals("none"))
{
Logger.Information(
$"About to add Emissions Region and Mine Carbon Emissions Data for {regionWattTimeName} from WattTime URL {wattTimeApiUrl} from {historicStartDateTime} to {historicEndDateTime} for historic data and insert them into the database",
"ApiDataMiner.MineRegionData()");
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
emissionsRegionId =
_objectModel.AddEmissionsRegion(friendlyName, timeZone, regionLat, regionLong,
regionWattTimeName)
.EmissionsRegionID;
CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
selfThrottlingMethod,
this.DatabaseConnectionString,
maxNumberOfCallsPerMinute);
// Mine Recent Actual Data
carbonEmissionsMiner.MineHistoricCarbonResults(
historicStartDateTime,
historicEndDateTime,
regionWattTimeName,
(int)emissionsRegionId);
// Mine Forecast Data
carbonEmissionsMiner.MineForecastMarginalCarbonResults(
forecastStartDateTime,
forecastEndDateTime,
regionWattTimeName,
(int)emissionsRegionId);
}
}
else
{
Logger.Information(
$"No WattTime Api Key was specified. Skipping this region for Emissions.",
"RunAsync()");
}
}
}
// Mine the regions weather if a weather node was supplied
int?weatherRegionId = null;
if (regionConfiguration.WeatherMiningRegion != null)
{
var friendlyName = regionConfiguration.WeatherMiningRegion.friendlyName;
using (
new TimedOperation(
$"Beginning Mining of weather data for Region {friendlyName}",
"ApiDataMiner.MineRegionData()"))
{
var timeZone = regionConfiguration.WeatherMiningRegion.TimeZone;
var regionLat = regionConfiguration.WeatherMiningRegion.Latitude;
var regionLong = regionConfiguration.WeatherMiningRegion.Longitude;
var weatherRegionWundergroundSubUrl =
regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl;
var wundergroundApiUrl = regionConfiguration.WeatherMiningRegion.ApiUrl;
string wundergroundApiKey = null;
if (string.IsNullOrEmpty(wundergroundApiKeyOverride) || wundergroundApiKeyOverride.Equals("none"))
{
wundergroundApiKey = regionConfiguration.WeatherMiningRegion.ApiKey;
}
else
{
wundergroundApiKey = wundergroundApiKeyOverride;
}
var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;
var historicStartDateTime = DateTime.UtcNow.AddDays(-1);
var historicEndDateTime = DateTime.UtcNow.AddDays(1);
if (!string.IsNullOrEmpty(wundergroundApiKey) && !wundergroundApiKey.Equals("none"))
{
Logger.Information(
$"About to add Emissions Region and Mine Carbon Emissions Data for {friendlyName} from Wunderground URL {weatherRegionWundergroundSubUrl} from {historicStartDateTime} to {historicEndDateTime} for historic data and insert them into the database",
"ApiDataMiner.MineRegionData()");
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
weatherRegionId =
_objectModel.AddWeatherRegion(
friendlyName,
timeZone,
regionLat,
regionLong,
weatherRegionWundergroundSubUrl).WeatherRegionID;
WeatherDataMiner weatherDataMiner = new WeatherDataMiner(
wundergroundApiUrl,
wundergroundApiKey,
selfThrottlingMethod,
this.DatabaseConnectionString,
maxNumberOfCallsPerMinute);
switch (regionConfiguration.WeatherMiningRegion.MiningMethod)
{
case "GPS":
// Mine Recent Actual Data
weatherDataMiner.MineHistoricWeatherValues(
historicStartDateTime,
historicEndDateTime,
regionLat,
regionLong,
(int)weatherRegionId);
// Mine Forecast Data
weatherDataMiner.MineTenDayHourlyForecastWeatherValues(
regionLat,
regionLong,
(int)weatherRegionId);
break;
case "WundergroundPageSubUrl":
default:
// Mine Recent Actual Data
weatherDataMiner.MineHistoricWeatherValues(
historicStartDateTime,
historicEndDateTime,
weatherRegionWundergroundSubUrl,
(int)weatherRegionId);
// Mine Forecast Data
weatherDataMiner.MineTenDayHourlyForecastWeatherValues(
weatherRegionWundergroundSubUrl,
(int)weatherRegionId);
break;
}
}
}
else
{
Logger.Information(
$"No Wunderground Api Key was specified. Skipping this region for Weather.",
"RunAsync()");
}
}
}
// Group the Emissions and Weather regions if they haven't already been grouped
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
_objectModel.AddMarketWeatherEmissionsRegionMapping(
regionGroupingName,
null,
weatherRegionId,
emissionsRegionId);
}
}
}