/// <summary>
/// Take a config element containing the data for a region to be mined, mine the data and save it in the database
/// </summary>
/// <param name="regionConfiguration"></param>
public void MineRegionData(ApiMinerConfigLayoutRegion regionConfiguration)
{
var regionGroupingName = regionConfiguration.friendlyName;
using (new TimedOperation(
$"Beginning Mining of all data for Region {regionGroupingName}",
"ApiDataMiner.MineRegionData()"))
{
// Mine the regions emissions if an emissions node was supplied
int?emissionsRegionId = null;
if (regionConfiguration.EmissionsMiningRegion != null)
{
var friendlyName = regionConfiguration.EmissionsMiningRegion.friendlyName;
using (
new TimedOperation(
$"Beginning Mining of emissions data for Region {friendlyName}",
"ApiDataMiner.MineRegionData()"))
{
var timeZone = regionConfiguration.EmissionsMiningRegion.TimeZone;
var regionLat = regionConfiguration.EmissionsMiningRegion.Latitude;
var regionLong = regionConfiguration.EmissionsMiningRegion.Longitude;
var regionWattTimeName = regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation;
var wattTimeApiUrl = regionConfiguration.EmissionsMiningRegion.ApiUrl;
var wattTimeApiKey = regionConfiguration.EmissionsMiningRegion.ApiKey;
var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;
var historicStartDateTime = DateTime.UtcNow.AddDays(-15);
var historicEndDateTime = DateTime.UtcNow.AddDays(1);
var forecastStartDateTime = DateTime.UtcNow.AddDays(-2);
var forecastEndDateTime = DateTime.UtcNow.AddDays(10);
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
emissionsRegionId =
_objectModel.AddEmissionsRegion(friendlyName, timeZone, regionLat, regionLong)
.EmissionsRegionID;
CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
selfThrottlingMethod,
this.DatabaseConnectionString,
maxNumberOfCallsPerMinute);
// Mine Recent Actual Data
carbonEmissionsMiner.MineHistoricCarbonResults(
historicStartDateTime,
historicEndDateTime,
regionWattTimeName,
(int)emissionsRegionId);
// Mine Forecast Data
carbonEmissionsMiner.MineForecastMarginalCarbonResults(
forecastStartDateTime,
forecastEndDateTime,
regionWattTimeName,
(int)emissionsRegionId);
}
}
}
// Mine the regions emissions if an emissions node was supplied
int?weatherRegionId = null;
if (regionConfiguration.WeatherMiningRegion != null)
{
var friendlyName = regionConfiguration.WeatherMiningRegion.friendlyName;
using (
new TimedOperation(
$"Beginning Mining of weather data for Region {friendlyName}",
"ApiDataMiner.MineRegionData()"))
{
var timeZone = regionConfiguration.WeatherMiningRegion.TimeZone;
var regionLat = regionConfiguration.WeatherMiningRegion.Latitude;
var regionLong = regionConfiguration.WeatherMiningRegion.Longitude;
var weatherRegionWundergroundSubUrl =
regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl;
var wundergroundApiUrl = regionConfiguration.WeatherMiningRegion.ApiUrl;
var wundergroundApiKey = regionConfiguration.WeatherMiningRegion.ApiKey;
var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;
var historicStartDateTime = DateTime.UtcNow.AddDays(-1);
var historicEndDateTime = DateTime.UtcNow.AddDays(1);
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
weatherRegionId =
_objectModel.AddWeatherRegion(
friendlyName,
timeZone,
regionLat,
regionLong,
weatherRegionWundergroundSubUrl).WeatherRegionID;
WeatherDataMiner weatherDataMiner = new WeatherDataMiner(
wundergroundApiUrl,
wundergroundApiKey,
selfThrottlingMethod,
this.DatabaseConnectionString,
maxNumberOfCallsPerMinute);
// Mine Recent Actual Data
weatherDataMiner.MineHistoricWeatherValues(
historicStartDateTime,
historicEndDateTime,
weatherRegionWundergroundSubUrl,
(int)weatherRegionId);
// Mine Forecast Data
weatherDataMiner.MineTenDayHourlyForecastWeatherValues(
weatherRegionWundergroundSubUrl,
(int)weatherRegionId);
}
}
}
// Group the Emissions and Weather regions if they haven't already been grouped
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
_objectModel.AddMarketWeatherEmissionsRegionMapping(
regionGroupingName,
null,
weatherRegionId,
emissionsRegionId);
}
}
}
/// <summary>
/// Take a config element containing the data for a region to be mined, mine the data and save it in the database
/// </summary>
/// <param name="regionConfiguration"></param>
/// <param name="wattTimeApiKeyOverride">Optional wattTimeApiKey to Override what's in the MinerConfigXML File</param>
/// <param name="wundergroundApiKeyOverride">Optional WundergroundApiKey to Override what's in the MinerConfigXML File</param>
public void MineRegionData(ApiMinerConfigLayoutRegion regionConfiguration, string wattTimeApiKeyOverride = null,
string wundergroundApiKeyOverride = null)
{
var regionGroupingName = regionConfiguration.friendlyName;
using (new TimedOperation(
$"Beginning Mining of all data for Region {regionGroupingName}",
"ApiDataMiner.MineRegionData()"))
{
// Mine the regions emissions if an emissions node was supplied
int?emissionsRegionId = null;
if (regionConfiguration.EmissionsMiningRegion != null)
{
var friendlyName = regionConfiguration.EmissionsMiningRegion.friendlyName;
using (
new TimedOperation(
$"Beginning Mining of emissions data for Region {friendlyName}",
"ApiDataMiner.MineRegionData()"))
{
var timeZone = regionConfiguration.EmissionsMiningRegion.TimeZone;
var regionLat = regionConfiguration.EmissionsMiningRegion.Latitude;
var regionLong = regionConfiguration.EmissionsMiningRegion.Longitude;
var regionWattTimeName =
regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation;
var wattTimeApiUrl = regionConfiguration.EmissionsMiningRegion.ApiUrl;
string wattTimeApiKey = null;
if (string.IsNullOrEmpty(wattTimeApiKeyOverride) || wattTimeApiKeyOverride.Equals("none"))
{
wattTimeApiKey = regionConfiguration.EmissionsMiningRegion.ApiKey;
}
else
{
wattTimeApiKey = wattTimeApiKeyOverride;
}
var selfThrottlingMethod = regionConfiguration.EmissionsMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.EmissionsMiningRegion.MaxNumberOfCallsPerMinute;
var historicStartDateTime = DateTime.UtcNow.AddDays(-15);
var historicEndDateTime = DateTime.UtcNow.AddDays(1);
var forecastStartDateTime = DateTime.UtcNow.AddDays(-2);
var forecastEndDateTime = DateTime.UtcNow.AddDays(10);
if (!string.IsNullOrEmpty(wattTimeApiKey) && !wattTimeApiKey.Equals("none"))
{
Logger.Information(
$"About to add Emissions Region and Mine Carbon Emissions Data for {regionWattTimeName} from WattTime URL {wattTimeApiUrl} from {historicStartDateTime} to {historicEndDateTime} for historic data and insert them into the database",
"ApiDataMiner.MineRegionData()");
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
emissionsRegionId =
_objectModel.AddEmissionsRegion(friendlyName, timeZone, regionLat, regionLong,
regionWattTimeName)
.EmissionsRegionID;
CarbonEmissionsMiner carbonEmissionsMiner = new CarbonEmissionsMiner(
wattTimeApiUrl,
wattTimeApiKey,
selfThrottlingMethod,
this.DatabaseConnectionString,
maxNumberOfCallsPerMinute);
// Mine Recent Actual Data
carbonEmissionsMiner.MineHistoricCarbonResults(
historicStartDateTime,
historicEndDateTime,
regionWattTimeName,
(int)emissionsRegionId);
// Mine Forecast Data
carbonEmissionsMiner.MineForecastMarginalCarbonResults(
forecastStartDateTime,
forecastEndDateTime,
regionWattTimeName,
(int)emissionsRegionId);
}
}
else
{
Logger.Information(
$"No WattTime Api Key was specified. Skipping this region for Emissions.",
"RunAsync()");
}
}
}
// Mine the regions weather if a weather node was supplied
int?weatherRegionId = null;
if (regionConfiguration.WeatherMiningRegion != null)
{
var friendlyName = regionConfiguration.WeatherMiningRegion.friendlyName;
using (
new TimedOperation(
$"Beginning Mining of weather data for Region {friendlyName}",
"ApiDataMiner.MineRegionData()"))
{
var timeZone = regionConfiguration.WeatherMiningRegion.TimeZone;
var regionLat = regionConfiguration.WeatherMiningRegion.Latitude;
var regionLong = regionConfiguration.WeatherMiningRegion.Longitude;
var weatherRegionWundergroundSubUrl =
regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl;
var wundergroundApiUrl = regionConfiguration.WeatherMiningRegion.ApiUrl;
string wundergroundApiKey = null;
if (string.IsNullOrEmpty(wundergroundApiKeyOverride) || wundergroundApiKeyOverride.Equals("none"))
{
wundergroundApiKey = regionConfiguration.WeatherMiningRegion.ApiKey;
}
else
{
wundergroundApiKey = wundergroundApiKeyOverride;
}
var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;
var historicStartDateTime = DateTime.UtcNow.AddDays(-1);
var historicEndDateTime = DateTime.UtcNow.AddDays(1);
if (!string.IsNullOrEmpty(wundergroundApiKey) && !wundergroundApiKey.Equals("none"))
{
Logger.Information(
$"About to add Emissions Region and Mine Carbon Emissions Data for {friendlyName} from Wunderground URL {weatherRegionWundergroundSubUrl} from {historicStartDateTime} to {historicEndDateTime} for historic data and insert them into the database",
"ApiDataMiner.MineRegionData()");
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
weatherRegionId =
_objectModel.AddWeatherRegion(
friendlyName,
timeZone,
regionLat,
regionLong,
weatherRegionWundergroundSubUrl).WeatherRegionID;
WeatherDataMiner weatherDataMiner = new WeatherDataMiner(
wundergroundApiUrl,
wundergroundApiKey,
selfThrottlingMethod,
this.DatabaseConnectionString,
maxNumberOfCallsPerMinute);
switch (regionConfiguration.WeatherMiningRegion.MiningMethod)
{
case "GPS":
// Mine Recent Actual Data
weatherDataMiner.MineHistoricWeatherValues(
historicStartDateTime,
historicEndDateTime,
regionLat,
regionLong,
(int)weatherRegionId);
// Mine Forecast Data
weatherDataMiner.MineTenDayHourlyForecastWeatherValues(
regionLat,
regionLong,
(int)weatherRegionId);
break;
case "WundergroundPageSubUrl":
default:
// Mine Recent Actual Data
weatherDataMiner.MineHistoricWeatherValues(
historicStartDateTime,
historicEndDateTime,
weatherRegionWundergroundSubUrl,
(int)weatherRegionId);
// Mine Forecast Data
weatherDataMiner.MineTenDayHourlyForecastWeatherValues(
weatherRegionWundergroundSubUrl,
(int)weatherRegionId);
break;
}
}
}
else
{
Logger.Information(
$"No Wunderground Api Key was specified. Skipping this region for Weather.",
"RunAsync()");
}
}
}
// Group the Emissions and Weather regions if they haven't already been grouped
using (var _objectModel = new SmartEnergyOM(this.DatabaseConnectionString))
{
_objectModel.AddMarketWeatherEmissionsRegionMapping(
regionGroupingName,
null,
weatherRegionId,
emissionsRegionId);
}
}
}