public void Add(UtcDateTimeEnumerator enumer,
SortedDictionary<UtcDateTime, double?> sumPowerInParkAtTimeTAtTimeT,
SortedDictionary<UtcDateTime, double?> sumObsPowerInParkAtTimeT,
int hoursAheadOffset)
{
nrOfRegisteredForecastTimePoints++;
int index = hoursAheadOffset;
foreach (UtcDateTime time in enumer)
{
if (sumPowerInParkAtTimeTAtTimeT.ContainsKey(time) && sumObsPowerInParkAtTimeT.ContainsKey(time))
{
if (sumPowerInParkAtTimeTAtTimeT[time].HasValue && sumObsPowerInParkAtTimeT[time].HasValue)
{
if (skillScoreHourBins.ContainsKey(Any))
skillScoreHourBins[Any].Register(sumObsPowerInParkAtTimeT[time].Value, sumPowerInParkAtTimeTAtTimeT[time].Value);
int hourAhead = index;
if (skillScoreHourBins.ContainsKey(hourAhead))
{
skillScoreHourBins[hourAhead].Register(sumObsPowerInParkAtTimeT[time].Value, sumPowerInParkAtTimeTAtTimeT[time].Value);
}
}
}
index++;
}
}
internal static SortedDictionary<UtcDateTime, double?> CalculateMarketResolutionAverage(UtcDateTime firstTimePoint, UtcDateTime first, UtcDateTime last, TimeSpan marketResolution, TimeSpan dataResolution, SortedDictionary<UtcDateTime, double?> prod, bool isForecast=false)
{
int steps;
TimeResolutionType timeResolutionType = dataResolution.ToTimeResolution(out steps);
// Create TimePoints that we are interested to aggregate data on:
var time = last;//.Subtract(marketResolution);
var timePoints = new List<UtcDateTime> { last };
do
{
time = time.Subtract(marketResolution);
if (time >= first) timePoints.Add(time);
} while (time >= first);
var resolutionSamples = new SortedDictionary<UtcDateTime, List<double?>>();
for (int index = 0; index < timePoints.Count; index++)
{
if (index + 1 < timePoints.Count)
{
UtcDateTime point = timePoints[index]; // this should be end time e.g 12:00
UtcDateTime prevPoint = timePoints[index + 1].Add(dataResolution); // this should be time resolution before, e.g. an hour, thus 11:00, but maybe adding data resolution e.g. 10.
var enumer = new UtcDateTimeEnumerator(prevPoint, point, timeResolutionType, steps);
foreach (UtcDateTime utcDateTime in enumer)
{
if (prod.ContainsKey(utcDateTime))
{
double? production = prod[utcDateTime];
if (!resolutionSamples.ContainsKey(point)) resolutionSamples.Add(point, new List<double?>());
resolutionSamples[point].Add(production);
}
}
}
else // This is for the last point...
{
UtcDateTime point = timePoints[index]; // this should be end time e.g 12:00
UtcDateTime prevPoint = timePoints[index].Subtract(marketResolution).Add(dataResolution); // this should be time resolution before, e.g. an hour, thus 11:00, but maybe adding data resolution e.g. 10.
var enumer = new UtcDateTimeEnumerator(prevPoint, point, timeResolutionType, steps);
foreach (UtcDateTime utcDateTime in enumer)
{
if (prod.ContainsKey(utcDateTime))
{
double? production = prod[utcDateTime];
if (!resolutionSamples.ContainsKey(point)) resolutionSamples.Add(point, new List<double?>());
resolutionSamples[point].Add(production);
}
}
}
}
int productionStepsInResolution = CalculateProductionStepsInResolution(marketResolution, dataResolution);
var avgProds = new SortedDictionary<UtcDateTime, double?>();
foreach (UtcDateTime dateTime in resolutionSamples.Keys)
{
var prodInResolution = resolutionSamples[dateTime];
// When calculating observed production, we always calculate the average as sum devided by time steps in period (resolution)
double? avgPower;
if (!isForecast) avgPower = prodInResolution.Sum()/productionStepsInResolution;
else
{ // When calculating predicted production, we always calculate the pure average
double sum = 0;
int nrOfObsInPeriod = 0;
foreach (double? p in prodInResolution)
{
if (p.HasValue)
{
nrOfObsInPeriod++;
sum += p.Value;
}
}
if (nrOfObsInPeriod > 0) avgPower = sum/nrOfObsInPeriod;
else avgPower = null;
}
avgProds.Add(dateTime, avgPower);
}
return avgProds;
}
public static ConcurrentDictionary<UtcDateTime, HourlySkillScoreCalculator> Calculate(ForecastMetaData fmd, string path, ObservationMetaData omd, string file, int[] scope,
bool includeNegObs = false, bool useFixedHours = false, string siteId=null)
{
var startTimesInMinutes = FindUsualForecastStartTimePoints(path, fmd);
var results = new ConcurrentDictionary<UtcDateTime, HourlySkillScoreCalculator>();
bool isNotPointingToANumber = false;
// 1. Load observations
string[] obsLines = File.ReadAllLines(file);
var obsPowerInFarm = new SortedDictionary<UtcDateTime, double?>();
string dateTimePattern = string.Empty;
string prevDateTimePattern = string.Empty;
if (!obsLines[0].Contains(omd.ObservationSep)) throw new Exception("Observations series does not contain the char: '" + omd.ObservationSep + "' \ras column seperator. Open the document and check what \rdelimiter char is used to seperate the columns \rand specify this char in the Observations Column Seperator field.");
for (int i=1; i<obsLines.Length; i++) // Skip the heading...
{
var line = obsLines[i];
var cols = line.Split(omd.ObservationSep);
if (cols.Length > 1)
{
string timeStampStr = cols[omd.ObservationTimeIndex];
string valueStr = cols[omd.ObservationValueIndex];
UtcDateTime? dateTime = DateTimeUtil.ParseTimeStamp(timeStampStr, prevDateTimePattern, out dateTimePattern);
if (dateTime.HasValue)
{
if (string.IsNullOrWhiteSpace(prevDateTimePattern) && !string.IsNullOrWhiteSpace(dateTimePattern)) prevDateTimePattern = dateTimePattern;
double? value = DoubleUtil.ParseDoubleValue(valueStr);
if (value.HasValue)
{
double val = value.Value;
if (omd.ObservationUnitType == "MW") val *= 1000;
if (val < 0 && !includeNegObs) obsPowerInFarm.Add(dateTime.Value, null);
else obsPowerInFarm.Add(dateTime.Value, val);
}
else
{
if (i <= 3) isNotPointingToANumber = true;
obsPowerInFarm.Add(dateTime.Value, null);
}
}
}
}
if (isNotPointingToANumber && obsPowerInFarm.Count == 0) throw new Exception("The Observation Column Index Value is not pointing to a column which contains a double value. Change index value!");
int obsSteps;
TimeResolutionType? obsResolution = IdentifyResolution(obsPowerInFarm, out obsSteps);
// 2. Identify Time Resolution of obs series, if less than an hour, make it hourly avg.
var firstObs = obsPowerInFarm.First().Key;
var minutes = startTimesInMinutes.Keys.ToArray();
foreach (var minute in minutes)
{
if (firstObs.Minute == minute)
{
SortedDictionary<UtcDateTime, double?> convertedObsPowerInFarm = ConvertTimeSeriesToHourlyResolution(firstObs, obsPowerInFarm, useFixedHours);
startTimesInMinutes[minute] = convertedObsPowerInFarm;
}
else if (minute > 0)
{
var first = new UtcDateTime(firstObs.Year, firstObs.Month, firstObs.Day, firstObs.Hour, minute);
SortedDictionary<UtcDateTime, double?> convertedObsPowerInFarm = ConvertTimeSeriesToHourlyResolution(first, obsPowerInFarm, useFixedHours, minute);
startTimesInMinutes[minute] = convertedObsPowerInFarm;
}
else
{
var tmp = firstObs.AddHours(1);
firstObs = new UtcDateTime(tmp.Year, tmp.Month, tmp.Day, tmp.Hour);
SortedDictionary<UtcDateTime, double?> convertedObsPowerInFarm = ConvertTimeSeriesToHourlyResolution(firstObs, obsPowerInFarm, useFixedHours);
startTimesInMinutes[minute] = convertedObsPowerInFarm;
}
}
// 3. Search forecastPath for forecast documents...
dateTimePattern = string.Empty;
prevDateTimePattern = string.Empty;
FileInfo[] files;
string filter = "*.csv";
if (Directory.Exists(path))
{
if (!string.IsNullOrEmpty(fmd.ForecastFileFilter)) filter = fmd.ForecastFileFilter;
var dir = new DirectoryInfo(path);
files = dir.GetFiles(filter, SearchOption.AllDirectories);
}
else files = new FileInfo[]{new FileInfo(path) };
// If we only have one single forecast file, then this forecast file is usually 99% of the time a long
// time series with continous time steps, meaning there are no overlaps. Usually, only the unique hours
// from the original forecasts have been extracted. In these situations, we are interested in comparing
// the skill of all hours and only use a single skill-bucket. Instead of making this an
// explicit setting in the view (which it should be ultimately), it is no supporting this case by
// convention. This should be changed later for flexibility...
bool continousSerie = files.Length == 1;
prevDateTimePattern = string.Empty;
foreach (var fileInfo in files)
{
isNotPointingToANumber = false;
UtcDateTime? firstTimePoint = null;
UtcDateTime? firstPossibleHour = null;
SortedDictionary<UtcDateTime, double?> predPowerInFarm;
var stream = fileInfo.OpenRead();
using (var reader = new StreamReader(stream))
{
predPowerInFarm = ReadPredictionFile(reader, fmd, includeNegObs, useFixedHours, siteId, prevDateTimePattern, out firstTimePoint, out firstPossibleHour, out isNotPointingToANumber);
}
if (isNotPointingToANumber) throw new Exception("The Forecast Column Index Value is not pointing to a column which contains a double value. Change index value!");
if (predPowerInFarm.Count == 0) continue;
if (!firstPossibleHour.HasValue) throw new Exception("The first possible forecast hour could not be determined. Please check forecast file: " + fileInfo.FullName);
if (!firstTimePoint.HasValue) throw new Exception("First time point in forecast file is not specified. Please check forecast file: " + fileInfo.FullName);
if (obsResolution.HasValue)
{
var firstForecastHour = firstTimePoint.Value;
if (useFixedHours) firstForecastHour = firstPossibleHour.Value;
var convertedObsPowerInFarm = startTimesInMinutes[firstForecastHour.Minute];
var convertedPredPowerInFarm = ConvertTimeSeriesToHourlyResolution(firstForecastHour, predPowerInFarm, useFixedHours, firstForecastHour.Minute);
if (convertedPredPowerInFarm.Count > 0)
{
UtcDateTime first = convertedPredPowerInFarm.Keys.First();
var firstTimePointInMonth = new UtcDateTime(first.Year, first.Month, 1);
if (continousSerie)
{
// Split up into months
UtcDateTime last = convertedPredPowerInFarm.Keys.Last();
var firstTimePointInLastMonth = new UtcDateTime(last.Year, last.Month, 1);
for (var time = firstTimePointInMonth; time.UtcTime <= firstTimePointInLastMonth.UtcTime; time = new UtcDateTime(time.UtcTime.AddMonths(1)))
{
if (!results.ContainsKey(time))
results.TryAdd(time, new HourlySkillScoreCalculator(new List<Turbine>(), scope, (int) omd.NormalizationValue));
HourlySkillScoreCalculator skillCalculator = results[time];
var enumer = new UtcDateTimeEnumerator(time, new UtcDateTime(time.UtcTime.AddMonths(1)), TimeResolution.Hour);
skillCalculator.AddContinousSerie(enumer, convertedPredPowerInFarm, convertedObsPowerInFarm);
}
}
else
{
if (!results.ContainsKey(firstTimePointInMonth))
results.TryAdd(firstTimePointInMonth, new HourlySkillScoreCalculator(new List<Turbine>(), scope, (int) omd.NormalizationValue));
HourlySkillScoreCalculator skillCalculator = results[firstTimePointInMonth];
var enumer = new UtcDateTimeEnumerator(first, convertedPredPowerInFarm.Keys.Last(), TimeResolution.Hour);
skillCalculator.Add(enumer, convertedPredPowerInFarm, convertedObsPowerInFarm, fmd.OffsetHoursAhead);
}
}
}
}
// 2.1 Print hourly obs series to file for end-user debugging and verification:
if(useFixedHours) PrintObsPowerInFarmToDebugFile(obsPowerInFarm);
return results;
}
public void Add(UtcDateTimeEnumerator enumer,
SortedDictionary<UtcDateTime, SortedDictionary<int, double>> powerInParkAtTimeT,
SortedDictionary<UtcDateTime, SortedDictionary<int, double>> obsPowerInParkAtTimeT,
SortedDictionary<UtcDateTime, double?> obsMastWeather)
{
nrOfRegisteredForecastTimePoints++;
var sumPowerInParkAtTimeTAtTimeT = new SortedDictionary<UtcDateTime, double>();
var sumObsPowerInParkAtTimeT = new SortedDictionary<UtcDateTime, double>();
UtcDateTime forecastStart = enumer.First();
var forecastSuccess = new SortedDictionary<UtcDateTime, RestrictedRangeInt>();
if (!Success.ContainsKey(forecastStart)) Success.Add(forecastStart, forecastSuccess);
double prevRotSpeed = -1;
int timeIndex = 0;
foreach (UtcDateTime currentTime in enumer)
{
forecastSuccess.Add(currentTime, new RestrictedRangeInt(0, turbines.Count(), 0));
if (timeIndex > 10 && sumObsPowerInParkAtTimeT.Count == 0)
{
break; // If we have nothing for the ten first hours then this forecast point is broken and should not be taken into account...
}
if (useObservedWeatherData)
{
if (obsMastWeather.Count == 0) throw new Exception("No MetMast Data provided, calculating skill score without met mast data must be implemented.");
}
bool isOkData = true;
int nrOfTubrines = 0;
if (useObservedWeatherData && obsMastWeather.ContainsKey(currentTime) && obsMastWeather[currentTime].HasValue && (obsMastWeather[currentTime].Value - prevRotSpeed).AboutEqual(0.0)) isOkData = false;
if (isOkData)
{
if (useObservedWeatherData && obsMastWeather.ContainsKey(currentTime) && obsMastWeather[currentTime].HasValue) prevRotSpeed = obsMastWeather[currentTime].Value;
if (powerInParkAtTimeT.ContainsKey(currentTime))
{
foreach (Turbine turbine in turbines)
{
if (obsPowerInParkAtTimeT.ContainsKey(currentTime) && obsPowerInParkAtTimeT[currentTime].ContainsKey(turbine.ScadaId))
{
double power = 0;
if (powerInParkAtTimeT.ContainsKey(currentTime) && powerInParkAtTimeT[currentTime].ContainsKey(turbine.ReferenceId)) power = powerInParkAtTimeT[currentTime][turbine.ReferenceId];
double obsPower = obsPowerInParkAtTimeT[currentTime][turbine.ScadaId];
if (!sumPowerInParkAtTimeTAtTimeT.ContainsKey(currentTime)) sumPowerInParkAtTimeTAtTimeT.Add(currentTime, 0);
sumPowerInParkAtTimeTAtTimeT[currentTime] += power;
if (!sumObsPowerInParkAtTimeT.ContainsKey(currentTime)) sumObsPowerInParkAtTimeT.Add(currentTime, 0);
sumObsPowerInParkAtTimeT[currentTime] += obsPower;
nrOfTubrines++;
}
}
}
}
forecastSuccess[currentTime].Value = nrOfTubrines;
timeIndex++;
}
int index = 0;
foreach (UtcDateTime time in enumer)
{
if (sumPowerInParkAtTimeTAtTimeT.ContainsKey(time) && sumObsPowerInParkAtTimeT.ContainsKey(time))
{
if (skillScoreHourBins.ContainsKey(Any)) skillScoreHourBins[Any].Register(sumObsPowerInParkAtTimeT[time], sumPowerInParkAtTimeTAtTimeT[time]);
int hourAhead = index;
if (skillScoreHourBins.ContainsKey(hourAhead))
{
skillScoreHourBins[hourAhead].Register(sumObsPowerInParkAtTimeT[time], sumPowerInParkAtTimeTAtTimeT[time]);
}
}
index++;
}
}
public void AddContinousSerie(UtcDateTimeEnumerator enumer, SortedDictionary<UtcDateTime, double?> predictionFrames, SortedDictionary<UtcDateTime, double?> productionFrames)
{
Dictionary<int, int> hourScope = Scope.ToDictionary(e => e, e => e);
nrOfRegisteredForecastTimePoints++;
double sumPower = 0;
double sumObsPower = 0;
double prevPower = -1;
int timeIndex = 0;
int regIndex = 0;
foreach (UtcDateTime currentTime in enumer)
{
if (timeIndex > 10 && sumObsPower <= 1)
{
break; // If we have nothing for the ten first hours then this forecast point is broken and should not be taken into account...
}
if (predictionFrames.ContainsKey(currentTime) && productionFrames.ContainsKey(currentTime)
&& predictionFrames[currentTime].HasValue && productionFrames[currentTime].HasValue)
{
double power = predictionFrames[currentTime].Value;
double obsPower = productionFrames[currentTime].Value;
sumPower += power;
sumObsPower += obsPower;
if(skillScoreHourBins.ContainsKey(Any)) skillScoreHourBins[Any].Register(obsPower, power);
int currentHour = currentTime.UtcTime.Hour;
if (hourScope.ContainsKey(currentHour))
{
int scopeHour = hourScope[currentHour];
if (skillScoreHourBins.ContainsKey(scopeHour))
{
skillScoreHourBins[scopeHour].Register(obsPower, power);
regIndex++;
}
}
prevPower = obsPower;
}
timeIndex++;
}
var first = enumer.First();
var suc = new SortedDictionary<UtcDateTime, RestrictedRangeInt>()
{
{first, new RestrictedRangeInt(0, timeIndex, regIndex)}
};
Success.Add(first, suc);
}
