private static void ParseLocalConfigAndAndMineData(string apiDataMinerConfigFileLocation)
{
var databaseConnectionString = ConfigurationManager.AppSettings["SQLAzureDatabaseEntityFrameworkConnectionString"];
var wattTimeApiKey = ConfigurationManager.AppSettings["wattTimeApiKey"];
var wundergroundApiKey = ConfigurationManager.AppSettings["wundergroundApiKey"];
var apiDataMiner = new ApiDataMiner(databaseConnectionString);
apiDataMiner.ParseMinerSettingsFileAndMineData(apiDataMinerConfigFileLocation, wattTimeApiKey, wundergroundApiKey);
}
private async Task RunAsync(CancellationToken cancellationToken)
{
Logger.Information("Data Miner Worker Role is Starting", "RunAsync()");
var databaseConnectionString = CloudConfigurationManager.GetSetting("SQLAzureDatabaseEntityFrameworkConnectionString");
var azureStorageConnectionString = CloudConfigurationManager.GetSetting("AzureStorageConnectionString");
var schedulerQueueName = CloudConfigurationManager.GetSetting("SchedulerQueueName");
var storageQueueManager = new AzureStorageQueueManager(azureStorageConnectionString, schedulerQueueName);
while (!cancellationToken.IsCancellationRequested)
{
// Check if there are any messages on the queue to instruct the data miner what to do
var message = storageQueueManager.GetMessage();
if (message != null)
{
// Instruction to run the data mining for the local config file
if (message.AsString.Contains("RunMinerForLocalConfigFile"))
{
// Parse config XML File defining regions, emissions regions, weather regions, etc.
using (
new TimedOperation(
"Beginning Mining of data pointed to by the configuration file",
"DataMinerWorkerRole.FullMinerCall"))
{
try
{
const string ConfigPath = @".\ApiDataMinerConfigs\ApiDataMinerConfigs.xml";
var apiDataMiner = new ApiDataMiner(databaseConnectionString);
apiDataMiner.ParseMinerSettingsFileAndMineData(ConfigPath);
}
catch (Exception exception)
{
Logger.Error(
"DataMinerWorkerRole encountered an exception mining all data pointed to by the configuration file",
"DataMinerWorkerRole.FullMinerCall",
null,
exception);
}
}
}
/* Add logic for any other mining here, and add a corresponding job to the Scheduler to send messages to the queue to kick it off */
}
// Sleep for a time before checking the queue again
var numberOfMinutesToSleep = 1;
Logger.Information($"Sleping for {numberOfMinutesToSleep} minutes", "RunAsync()");
await Task.Delay(1000 * 60 *numberOfMinutesToSleep, cancellationToken);
}
}
private static void ParseLocalConfigAndAndMineData(string apiDataMinerConfigFileLocation)
{
var databaseConnectionString = ConfigurationManager.AppSettings["SQLAzureDatabaseEntityFrameworkConnectionString"];
var wattTimeApiKey = ConfigurationManager.AppSettings["wattTimeApiKey"];
var wundergroundApiKey = ConfigurationManager.AppSettings["wundergroundApiKey"];
var wattTimeApiV2Url = ConfigurationManager.AppSettings["WattTimeApiV2Url"];
var wattTimeUsername = ConfigurationManager.AppSettings["WattTimeUsername"];
var wattTimePassword = ConfigurationManager.AppSettings["WattTimePassword"];
var wattTimeEmail = ConfigurationManager.AppSettings["WattTimeEmail"];
var wattTimeOrganization = ConfigurationManager.AppSettings["WattTimeOrganization"];
var darkSkyApiUrl = ConfigurationManager.AppSettings["DarkSkyApiUrl"];
var darkSkyApiKey = ConfigurationManager.AppSettings["DarkSkyApiKey"];
var apiDataMiner = new ApiDataMiner(databaseConnectionString);
apiDataMiner.ParseMinerSettingsFileAndMineData(apiDataMinerConfigFileLocation, wattTimeApiKey, wundergroundApiKey, darkSkyApiKey, wattTimeUsername, wattTimePassword, wattTimeEmail,
wattTimeOrganization);
}
public void TestParseMinerSettingsFileAndMineData()
{
// Arrange
var ConfigPath = @".\ApiDataMinerConfigs\ApiDataMinerConfigs.xml";
// Act
var apiDataMiner = new ApiDataMiner(databaseConnectionString);
apiDataMiner.ParseMinerSettingsFileAndMineData(ConfigPath);
// Assert
using (var streamReader = new StreamReader(ConfigPath))
{
var xmlSerializer = new XmlSerializer(typeof(ApiMinerConfigLayout));
var minerConfigs = (ApiMinerConfigLayout)xmlSerializer.Deserialize(streamReader);
foreach (var regionConfiguration in minerConfigs.Regions)
{
// Verify emissions were mined successfully for each region in the Config File
if (regionConfiguration.EmissionsMiningRegion != null)
{
var emissionsRegionName = regionConfiguration.EmissionsMiningRegion.friendlyName;
var timeZone = regionConfiguration.EmissionsMiningRegion.TimeZone;
var regionLat = regionConfiguration.EmissionsMiningRegion.Latitude;
var regionLong = regionConfiguration.EmissionsMiningRegion.Longitude;
var regionWattTimeName = regionConfiguration.EmissionsMiningRegion.EmissionsWattTimeAbbreviation;
var wattTimeApiUrl = regionConfiguration.EmissionsMiningRegion.ApiUrl;
var wattTimeApiKey = regionConfiguration.EmissionsMiningRegion.ApiKey;
var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;
var startDateTime = DateTime.UtcNow.AddDays(-2);
var endDateTime = DateTime.UtcNow.AddDays(10);
var wattTimeInteraction = new EmissionsApiInteraction(selfThrottlingMethod, maxNumberOfCallsPerMinute);
var results = wattTimeInteraction.GetObservedMarginalCarbonResults(
wattTimeApiUrl,
regionWattTimeName,
startDateTime,
endDateTime,
null,
wattTimeApiKey);
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddEmissionsRegion(emissionsRegionName, timeZone, regionLat, regionLong)
.EmissionsRegionID;
}
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
foreach (var result in results)
{
var dataPoint = _objectModel.FindCarbonEmissionsDataPoints(regionId, result.timestamp);
Assert.IsNotNull(dataPoint);
}
}
}
// Verify weather was mined successfully for each region in the Config File
if (regionConfiguration.WeatherMiningRegion != null)
{
var emissionsRegionName = regionConfiguration.WeatherMiningRegion.friendlyName;
var timeZone = regionConfiguration.WeatherMiningRegion.TimeZone;
var regionLat = regionConfiguration.WeatherMiningRegion.Latitude;
var regionLong = regionConfiguration.WeatherMiningRegion.Longitude;
var weatherRegionWundergroundSubUrl =
regionConfiguration.WeatherMiningRegion.weatherRegionWundergroundSubUrl;
var wundergroundApiUrl = regionConfiguration.WeatherMiningRegion.ApiUrl;
var wundergroundApiKey = regionConfiguration.WeatherMiningRegion.ApiKey;
var selfThrottlingMethod = regionConfiguration.WeatherMiningRegion.SelfThrottlingMethod;
var maxNumberOfCallsPerMinute =
regionConfiguration.WeatherMiningRegion.MaxNumberOfCallsPerMinute;
var wundergroundWeatherInteraction = new WundergroundWeatherInteraction(
selfThrottlingMethod,
maxNumberOfCallsPerMinute);
var results =
wundergroundWeatherInteraction.GetTenDayHourlyForecastWeatherData(
wundergroundApiUrl,
weatherRegionWundergroundSubUrl,
wundergroundApiKey);
int regionId;
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
regionId =
_objectModel.AddWeatherRegion(emissionsRegionName, timeZone, regionLat, regionLong)
.WeatherRegionID;
}
using (var _objectModel = new SmartEnergyOM(databaseConnectionString))
{
foreach (var result in results)
{
var dataPoint = _objectModel.FindWeatherDataPoints(
regionId,
result.observationDateTime);
Assert.IsNotNull(dataPoint);
}
}
}
}
}
}
