private static SortedDictionary<int, SortedDictionary<UtcDateTime, double?>> FindUsualForecastStartTimePoints(string path, ForecastMetaData fmd)
{
var forecastStartInMinutesOverTheHour = new SortedDictionary<int, SortedDictionary<UtcDateTime, double?>>();
FileInfo[] files;
string filter = "*.csv";
if (Directory.Exists(path))
{
if (!string.IsNullOrEmpty(fmd.ForecastFileFilter)) filter = fmd.ForecastFileFilter;
var dir = new DirectoryInfo(path);
files = dir.GetFiles(filter, SearchOption.AllDirectories).Take(10).ToArray();
}
else files = new FileInfo[] { new FileInfo(path) };
string prevDateTimePattern = string.Empty;
foreach (var fileInfo in files)
{
bool isNotPointingToANumber = false;
UtcDateTime? firstTimePoint = null;
UtcDateTime? firstPossibleHour = null;
SortedDictionary<UtcDateTime, double?> predPowerInFarm;
var stream = fileInfo.OpenRead();
using (var reader = new StreamReader(stream))
{
predPowerInFarm = ReadPredictionFile(reader, fmd, true, false, "N/A", prevDateTimePattern, out firstTimePoint, out firstPossibleHour, out isNotPointingToANumber);
}
if (predPowerInFarm.Any())
{
int minutes = predPowerInFarm.First().Key.Minute;
if(!forecastStartInMinutesOverTheHour.ContainsKey(minutes)) forecastStartInMinutesOverTheHour.Add(minutes, null);
}
}
return forecastStartInMinutesOverTheHour;
}
private static SortedDictionary<UtcDateTime, double?> ReadPredictionFile(StreamReader reader, ForecastMetaData fmd, bool includeNegObs, bool useFixedHours, string siteId, string prevDateTimePattern,
out UtcDateTime? firstTimePoint,
out UtcDateTime? firstPossibleHour,
out bool isNotPointingToANumber)
{
isNotPointingToANumber = false;
firstTimePoint = null;
firstPossibleHour = null;
int lineNr = -1;
string dateTimePattern;
var predPowerInFarm = new SortedDictionary<UtcDateTime, double?>();
while (!reader.EndOfStream)
{
string line = reader.ReadLine();
if (lineNr == -1)
{
if (!line.Contains(fmd.ForecastSep))
throw new Exception("Forecast series does not contain the char: '" + fmd.ForecastSep +
"' \ras column seperator. Open a document and check what \rdelimiter char is used to seperate the columns \rand specify this char in the Forecasts Column Seperator field.");
lineNr++;
continue;
}
if (string.IsNullOrEmpty(line)) continue;
var cols = line.Split(fmd.ForecastSep);
bool collect = false;
if (!string.IsNullOrWhiteSpace(siteId))
{
collect = (cols.Length > 2 && cols[1].Equals(siteId, StringComparison.InvariantCultureIgnoreCase));
if (!collect && siteId.Equals("N/A", StringComparison.InvariantCultureIgnoreCase)) collect = true;
}
else collect = true;
string timeStampStr = cols[fmd.ForecastTimeIndex];
string valueStr = cols[fmd.ForecastValueIndex];
//int timeStepAhead = lineNr + fmd.OffsetHoursAhead;
// TODO: IF LINE STARTS WITH TEXT, CONTINUE
if (string.IsNullOrWhiteSpace(timeStampStr) || !char.IsNumber(timeStampStr[0])) continue;
dateTimePattern = string.Empty;
UtcDateTime? dateTime = DateTimeUtil.ParseTimeStamp(timeStampStr, prevDateTimePattern, out dateTimePattern);
if (dateTime.HasValue)
{
if (string.IsNullOrWhiteSpace(prevDateTimePattern) && !string.IsNullOrWhiteSpace(dateTimePattern)) prevDateTimePattern = dateTimePattern;
// Hack for calc SWM perf, REMOVE
// if (lineNr == 0 && dateTime.Hour== 23) continue;
if (lineNr == 0)
{
firstTimePoint = dateTime;
firstPossibleHour = firstTimePoint;
if (useFixedHours && dateTime.HasValue)
{
var firstHour = dateTime.Value.AddHours(1);
firstPossibleHour = new UtcDateTime(firstHour.Year, firstHour.Month, firstHour.Day, firstHour.Hour);
}
}
if (collect)
{
double? value = DoubleUtil.ParseDoubleValue(valueStr);
if (value.HasValue)
{
double val = value.Value;
if (fmd.ForecastUnitType == "MW") val *= 1000;
if (val < 0 && !includeNegObs)
{
if(!predPowerInFarm.ContainsKey(dateTime.Value)) predPowerInFarm.Add(dateTime.Value, null);
}
else
{
if(!predPowerInFarm.ContainsKey(dateTime.Value)) predPowerInFarm.Add(dateTime.Value, val);
}
}
else
{
if (lineNr <= 3) isNotPointingToANumber = true;
else
{
if(!predPowerInFarm.ContainsKey(dateTime.Value)) predPowerInFarm.Add(dateTime.Value, null);
}
}
}
lineNr++; // We only count lines with valid time stamps
}
}
return predPowerInFarm;
}
public static ConcurrentDictionary<UtcDateTime, HourlySkillScoreCalculator> Calculate(ForecastMetaData fmd, string path, ObservationMetaData omd, string file, int[] scope,
bool includeNegObs = false, bool useFixedHours = false, string siteId=null)
{
var startTimesInMinutes = FindUsualForecastStartTimePoints(path, fmd);
var results = new ConcurrentDictionary<UtcDateTime, HourlySkillScoreCalculator>();
bool isNotPointingToANumber = false;
// 1. Load observations
string[] obsLines = File.ReadAllLines(file);
var obsPowerInFarm = new SortedDictionary<UtcDateTime, double?>();
string dateTimePattern = string.Empty;
string prevDateTimePattern = string.Empty;
if (!obsLines[0].Contains(omd.ObservationSep)) throw new Exception("Observations series does not contain the char: '" + omd.ObservationSep + "' \ras column seperator. Open the document and check what \rdelimiter char is used to seperate the columns \rand specify this char in the Observations Column Seperator field.");
for (int i=1; i<obsLines.Length; i++) // Skip the heading...
{
var line = obsLines[i];
var cols = line.Split(omd.ObservationSep);
if (cols.Length > 1)
{
string timeStampStr = cols[omd.ObservationTimeIndex];
string valueStr = cols[omd.ObservationValueIndex];
UtcDateTime? dateTime = DateTimeUtil.ParseTimeStamp(timeStampStr, prevDateTimePattern, out dateTimePattern);
if (dateTime.HasValue)
{
if (string.IsNullOrWhiteSpace(prevDateTimePattern) && !string.IsNullOrWhiteSpace(dateTimePattern)) prevDateTimePattern = dateTimePattern;
double? value = DoubleUtil.ParseDoubleValue(valueStr);
if (value.HasValue)
{
double val = value.Value;
if (omd.ObservationUnitType == "MW") val *= 1000;
if (val < 0 && !includeNegObs) obsPowerInFarm.Add(dateTime.Value, null);
else obsPowerInFarm.Add(dateTime.Value, val);
}
else
{
if (i <= 3) isNotPointingToANumber = true;
obsPowerInFarm.Add(dateTime.Value, null);
}
}
}
}
if (isNotPointingToANumber && obsPowerInFarm.Count == 0) throw new Exception("The Observation Column Index Value is not pointing to a column which contains a double value. Change index value!");
int obsSteps;
TimeResolutionType? obsResolution = IdentifyResolution(obsPowerInFarm, out obsSteps);
// 2. Identify Time Resolution of obs series, if less than an hour, make it hourly avg.
var firstObs = obsPowerInFarm.First().Key;
var minutes = startTimesInMinutes.Keys.ToArray();
foreach (var minute in minutes)
{
if (firstObs.Minute == minute)
{
SortedDictionary<UtcDateTime, double?> convertedObsPowerInFarm = ConvertTimeSeriesToHourlyResolution(firstObs, obsPowerInFarm, useFixedHours);
startTimesInMinutes[minute] = convertedObsPowerInFarm;
}
else if (minute > 0)
{
var first = new UtcDateTime(firstObs.Year, firstObs.Month, firstObs.Day, firstObs.Hour, minute);
SortedDictionary<UtcDateTime, double?> convertedObsPowerInFarm = ConvertTimeSeriesToHourlyResolution(first, obsPowerInFarm, useFixedHours, minute);
startTimesInMinutes[minute] = convertedObsPowerInFarm;
}
else
{
var tmp = firstObs.AddHours(1);
firstObs = new UtcDateTime(tmp.Year, tmp.Month, tmp.Day, tmp.Hour);
SortedDictionary<UtcDateTime, double?> convertedObsPowerInFarm = ConvertTimeSeriesToHourlyResolution(firstObs, obsPowerInFarm, useFixedHours);
startTimesInMinutes[minute] = convertedObsPowerInFarm;
}
}
// 3. Search forecastPath for forecast documents...
dateTimePattern = string.Empty;
prevDateTimePattern = string.Empty;
FileInfo[] files;
string filter = "*.csv";
if (Directory.Exists(path))
{
if (!string.IsNullOrEmpty(fmd.ForecastFileFilter)) filter = fmd.ForecastFileFilter;
var dir = new DirectoryInfo(path);
files = dir.GetFiles(filter, SearchOption.AllDirectories);
}
else files = new FileInfo[]{new FileInfo(path) };
// If we only have one single forecast file, then this forecast file is usually 99% of the time a long
// time series with continous time steps, meaning there are no overlaps. Usually, only the unique hours
// from the original forecasts have been extracted. In these situations, we are interested in comparing
// the skill of all hours and only use a single skill-bucket. Instead of making this an
// explicit setting in the view (which it should be ultimately), it is no supporting this case by
// convention. This should be changed later for flexibility...
bool continousSerie = files.Length == 1;
prevDateTimePattern = string.Empty;
foreach (var fileInfo in files)
{
isNotPointingToANumber = false;
UtcDateTime? firstTimePoint = null;
UtcDateTime? firstPossibleHour = null;
SortedDictionary<UtcDateTime, double?> predPowerInFarm;
var stream = fileInfo.OpenRead();
using (var reader = new StreamReader(stream))
{
predPowerInFarm = ReadPredictionFile(reader, fmd, includeNegObs, useFixedHours, siteId, prevDateTimePattern, out firstTimePoint, out firstPossibleHour, out isNotPointingToANumber);
}
if (isNotPointingToANumber) throw new Exception("The Forecast Column Index Value is not pointing to a column which contains a double value. Change index value!");
if (predPowerInFarm.Count == 0) continue;
if (!firstPossibleHour.HasValue) throw new Exception("The first possible forecast hour could not be determined. Please check forecast file: " + fileInfo.FullName);
if (!firstTimePoint.HasValue) throw new Exception("First time point in forecast file is not specified. Please check forecast file: " + fileInfo.FullName);
if (obsResolution.HasValue)
{
var firstForecastHour = firstTimePoint.Value;
if (useFixedHours) firstForecastHour = firstPossibleHour.Value;
var convertedObsPowerInFarm = startTimesInMinutes[firstForecastHour.Minute];
var convertedPredPowerInFarm = ConvertTimeSeriesToHourlyResolution(firstForecastHour, predPowerInFarm, useFixedHours, firstForecastHour.Minute);
if (convertedPredPowerInFarm.Count > 0)
{
UtcDateTime first = convertedPredPowerInFarm.Keys.First();
var firstTimePointInMonth = new UtcDateTime(first.Year, first.Month, 1);
if (continousSerie)
{
// Split up into months
UtcDateTime last = convertedPredPowerInFarm.Keys.Last();
var firstTimePointInLastMonth = new UtcDateTime(last.Year, last.Month, 1);
for (var time = firstTimePointInMonth; time.UtcTime <= firstTimePointInLastMonth.UtcTime; time = new UtcDateTime(time.UtcTime.AddMonths(1)))
{
if (!results.ContainsKey(time))
results.TryAdd(time, new HourlySkillScoreCalculator(new List<Turbine>(), scope, (int) omd.NormalizationValue));
HourlySkillScoreCalculator skillCalculator = results[time];
var enumer = new UtcDateTimeEnumerator(time, new UtcDateTime(time.UtcTime.AddMonths(1)), TimeResolution.Hour);
skillCalculator.AddContinousSerie(enumer, convertedPredPowerInFarm, convertedObsPowerInFarm);
}
}
else
{
if (!results.ContainsKey(firstTimePointInMonth))
results.TryAdd(firstTimePointInMonth, new HourlySkillScoreCalculator(new List<Turbine>(), scope, (int) omd.NormalizationValue));
HourlySkillScoreCalculator skillCalculator = results[firstTimePointInMonth];
var enumer = new UtcDateTimeEnumerator(first, convertedPredPowerInFarm.Keys.Last(), TimeResolution.Hour);
skillCalculator.Add(enumer, convertedPredPowerInFarm, convertedObsPowerInFarm, fmd.OffsetHoursAhead);
}
}
}
}
// 2.1 Print hourly obs series to file for end-user debugging and verification:
if(useFixedHours) PrintObsPowerInFarmToDebugFile(obsPowerInFarm);
return results;
}