private static SortedDictionary<int, SortedDictionary<UtcDateTime, double?>> FindUsualForecastStartTimePoints(string path, ForecastMetaData fmd) { var forecastStartInMinutesOverTheHour = new SortedDictionary<int, SortedDictionary<UtcDateTime, double?>>(); FileInfo[] files; string filter = "*.csv"; if (Directory.Exists(path)) { if (!string.IsNullOrEmpty(fmd.ForecastFileFilter)) filter = fmd.ForecastFileFilter; var dir = new DirectoryInfo(path); files = dir.GetFiles(filter, SearchOption.AllDirectories).Take(10).ToArray(); } else files = new FileInfo[] { new FileInfo(path) }; string prevDateTimePattern = string.Empty; foreach (var fileInfo in files) { bool isNotPointingToANumber = false; UtcDateTime? firstTimePoint = null; UtcDateTime? firstPossibleHour = null; SortedDictionary<UtcDateTime, double?> predPowerInFarm; var stream = fileInfo.OpenRead(); using (var reader = new StreamReader(stream)) { predPowerInFarm = ReadPredictionFile(reader, fmd, true, false, "N/A", prevDateTimePattern, out firstTimePoint, out firstPossibleHour, out isNotPointingToANumber); } if (predPowerInFarm.Any()) { int minutes = predPowerInFarm.First().Key.Minute; if(!forecastStartInMinutesOverTheHour.ContainsKey(minutes)) forecastStartInMinutesOverTheHour.Add(minutes, null); } } return forecastStartInMinutesOverTheHour; }
private static SortedDictionary<UtcDateTime, double?> ReadPredictionFile(StreamReader reader, ForecastMetaData fmd, bool includeNegObs, bool useFixedHours, string siteId, string prevDateTimePattern, out UtcDateTime? firstTimePoint, out UtcDateTime? firstPossibleHour, out bool isNotPointingToANumber) { isNotPointingToANumber = false; firstTimePoint = null; firstPossibleHour = null; int lineNr = -1; string dateTimePattern; var predPowerInFarm = new SortedDictionary<UtcDateTime, double?>(); while (!reader.EndOfStream) { string line = reader.ReadLine(); if (lineNr == -1) { if (!line.Contains(fmd.ForecastSep)) throw new Exception("Forecast series does not contain the char: '" + fmd.ForecastSep + "' \ras column seperator. Open a document and check what \rdelimiter char is used to seperate the columns \rand specify this char in the Forecasts Column Seperator field."); lineNr++; continue; } if (string.IsNullOrEmpty(line)) continue; var cols = line.Split(fmd.ForecastSep); bool collect = false; if (!string.IsNullOrWhiteSpace(siteId)) { collect = (cols.Length > 2 && cols[1].Equals(siteId, StringComparison.InvariantCultureIgnoreCase)); if (!collect && siteId.Equals("N/A", StringComparison.InvariantCultureIgnoreCase)) collect = true; } else collect = true; string timeStampStr = cols[fmd.ForecastTimeIndex]; string valueStr = cols[fmd.ForecastValueIndex]; //int timeStepAhead = lineNr + fmd.OffsetHoursAhead; // TODO: IF LINE STARTS WITH TEXT, CONTINUE if (string.IsNullOrWhiteSpace(timeStampStr) || !char.IsNumber(timeStampStr[0])) continue; dateTimePattern = string.Empty; UtcDateTime? dateTime = DateTimeUtil.ParseTimeStamp(timeStampStr, prevDateTimePattern, out dateTimePattern); if (dateTime.HasValue) { if (string.IsNullOrWhiteSpace(prevDateTimePattern) && !string.IsNullOrWhiteSpace(dateTimePattern)) prevDateTimePattern = dateTimePattern; // Hack for calc SWM perf, REMOVE // if (lineNr == 0 && dateTime.Hour== 23) continue; if (lineNr == 0) { firstTimePoint = dateTime; firstPossibleHour = firstTimePoint; if (useFixedHours && dateTime.HasValue) { var firstHour = dateTime.Value.AddHours(1); firstPossibleHour = new UtcDateTime(firstHour.Year, firstHour.Month, firstHour.Day, firstHour.Hour); } } if (collect) { double? value = DoubleUtil.ParseDoubleValue(valueStr); if (value.HasValue) { double val = value.Value; if (fmd.ForecastUnitType == "MW") val *= 1000; if (val < 0 && !includeNegObs) { if(!predPowerInFarm.ContainsKey(dateTime.Value)) predPowerInFarm.Add(dateTime.Value, null); } else { if(!predPowerInFarm.ContainsKey(dateTime.Value)) predPowerInFarm.Add(dateTime.Value, val); } } else { if (lineNr <= 3) isNotPointingToANumber = true; else { if(!predPowerInFarm.ContainsKey(dateTime.Value)) predPowerInFarm.Add(dateTime.Value, null); } } } lineNr++; // We only count lines with valid time stamps } } return predPowerInFarm; }
public static ConcurrentDictionary<UtcDateTime, HourlySkillScoreCalculator> Calculate(ForecastMetaData fmd, string path, ObservationMetaData omd, string file, int[] scope, bool includeNegObs = false, bool useFixedHours = false, string siteId=null) { var startTimesInMinutes = FindUsualForecastStartTimePoints(path, fmd); var results = new ConcurrentDictionary<UtcDateTime, HourlySkillScoreCalculator>(); bool isNotPointingToANumber = false; // 1. Load observations string[] obsLines = File.ReadAllLines(file); var obsPowerInFarm = new SortedDictionary<UtcDateTime, double?>(); string dateTimePattern = string.Empty; string prevDateTimePattern = string.Empty; if (!obsLines[0].Contains(omd.ObservationSep)) throw new Exception("Observations series does not contain the char: '" + omd.ObservationSep + "' \ras column seperator. Open the document and check what \rdelimiter char is used to seperate the columns \rand specify this char in the Observations Column Seperator field."); for (int i=1; i<obsLines.Length; i++) // Skip the heading... { var line = obsLines[i]; var cols = line.Split(omd.ObservationSep); if (cols.Length > 1) { string timeStampStr = cols[omd.ObservationTimeIndex]; string valueStr = cols[omd.ObservationValueIndex]; UtcDateTime? dateTime = DateTimeUtil.ParseTimeStamp(timeStampStr, prevDateTimePattern, out dateTimePattern); if (dateTime.HasValue) { if (string.IsNullOrWhiteSpace(prevDateTimePattern) && !string.IsNullOrWhiteSpace(dateTimePattern)) prevDateTimePattern = dateTimePattern; double? value = DoubleUtil.ParseDoubleValue(valueStr); if (value.HasValue) { double val = value.Value; if (omd.ObservationUnitType == "MW") val *= 1000; if (val < 0 && !includeNegObs) obsPowerInFarm.Add(dateTime.Value, null); else obsPowerInFarm.Add(dateTime.Value, val); } else { if (i <= 3) isNotPointingToANumber = true; obsPowerInFarm.Add(dateTime.Value, null); } } } } if (isNotPointingToANumber && obsPowerInFarm.Count == 0) throw new Exception("The Observation Column Index Value is not pointing to a column which contains a double value. Change index value!"); int obsSteps; TimeResolutionType? obsResolution = IdentifyResolution(obsPowerInFarm, out obsSteps); // 2. Identify Time Resolution of obs series, if less than an hour, make it hourly avg. var firstObs = obsPowerInFarm.First().Key; var minutes = startTimesInMinutes.Keys.ToArray(); foreach (var minute in minutes) { if (firstObs.Minute == minute) { SortedDictionary<UtcDateTime, double?> convertedObsPowerInFarm = ConvertTimeSeriesToHourlyResolution(firstObs, obsPowerInFarm, useFixedHours); startTimesInMinutes[minute] = convertedObsPowerInFarm; } else if (minute > 0) { var first = new UtcDateTime(firstObs.Year, firstObs.Month, firstObs.Day, firstObs.Hour, minute); SortedDictionary<UtcDateTime, double?> convertedObsPowerInFarm = ConvertTimeSeriesToHourlyResolution(first, obsPowerInFarm, useFixedHours, minute); startTimesInMinutes[minute] = convertedObsPowerInFarm; } else { var tmp = firstObs.AddHours(1); firstObs = new UtcDateTime(tmp.Year, tmp.Month, tmp.Day, tmp.Hour); SortedDictionary<UtcDateTime, double?> convertedObsPowerInFarm = ConvertTimeSeriesToHourlyResolution(firstObs, obsPowerInFarm, useFixedHours); startTimesInMinutes[minute] = convertedObsPowerInFarm; } } // 3. Search forecastPath for forecast documents... dateTimePattern = string.Empty; prevDateTimePattern = string.Empty; FileInfo[] files; string filter = "*.csv"; if (Directory.Exists(path)) { if (!string.IsNullOrEmpty(fmd.ForecastFileFilter)) filter = fmd.ForecastFileFilter; var dir = new DirectoryInfo(path); files = dir.GetFiles(filter, SearchOption.AllDirectories); } else files = new FileInfo[]{new FileInfo(path) }; // If we only have one single forecast file, then this forecast file is usually 99% of the time a long // time series with continous time steps, meaning there are no overlaps. Usually, only the unique hours // from the original forecasts have been extracted. In these situations, we are interested in comparing // the skill of all hours and only use a single skill-bucket. Instead of making this an // explicit setting in the view (which it should be ultimately), it is no supporting this case by // convention. This should be changed later for flexibility... bool continousSerie = files.Length == 1; prevDateTimePattern = string.Empty; foreach (var fileInfo in files) { isNotPointingToANumber = false; UtcDateTime? firstTimePoint = null; UtcDateTime? firstPossibleHour = null; SortedDictionary<UtcDateTime, double?> predPowerInFarm; var stream = fileInfo.OpenRead(); using (var reader = new StreamReader(stream)) { predPowerInFarm = ReadPredictionFile(reader, fmd, includeNegObs, useFixedHours, siteId, prevDateTimePattern, out firstTimePoint, out firstPossibleHour, out isNotPointingToANumber); } if (isNotPointingToANumber) throw new Exception("The Forecast Column Index Value is not pointing to a column which contains a double value. Change index value!"); if (predPowerInFarm.Count == 0) continue; if (!firstPossibleHour.HasValue) throw new Exception("The first possible forecast hour could not be determined. Please check forecast file: " + fileInfo.FullName); if (!firstTimePoint.HasValue) throw new Exception("First time point in forecast file is not specified. Please check forecast file: " + fileInfo.FullName); if (obsResolution.HasValue) { var firstForecastHour = firstTimePoint.Value; if (useFixedHours) firstForecastHour = firstPossibleHour.Value; var convertedObsPowerInFarm = startTimesInMinutes[firstForecastHour.Minute]; var convertedPredPowerInFarm = ConvertTimeSeriesToHourlyResolution(firstForecastHour, predPowerInFarm, useFixedHours, firstForecastHour.Minute); if (convertedPredPowerInFarm.Count > 0) { UtcDateTime first = convertedPredPowerInFarm.Keys.First(); var firstTimePointInMonth = new UtcDateTime(first.Year, first.Month, 1); if (continousSerie) { // Split up into months UtcDateTime last = convertedPredPowerInFarm.Keys.Last(); var firstTimePointInLastMonth = new UtcDateTime(last.Year, last.Month, 1); for (var time = firstTimePointInMonth; time.UtcTime <= firstTimePointInLastMonth.UtcTime; time = new UtcDateTime(time.UtcTime.AddMonths(1))) { if (!results.ContainsKey(time)) results.TryAdd(time, new HourlySkillScoreCalculator(new List<Turbine>(), scope, (int) omd.NormalizationValue)); HourlySkillScoreCalculator skillCalculator = results[time]; var enumer = new UtcDateTimeEnumerator(time, new UtcDateTime(time.UtcTime.AddMonths(1)), TimeResolution.Hour); skillCalculator.AddContinousSerie(enumer, convertedPredPowerInFarm, convertedObsPowerInFarm); } } else { if (!results.ContainsKey(firstTimePointInMonth)) results.TryAdd(firstTimePointInMonth, new HourlySkillScoreCalculator(new List<Turbine>(), scope, (int) omd.NormalizationValue)); HourlySkillScoreCalculator skillCalculator = results[firstTimePointInMonth]; var enumer = new UtcDateTimeEnumerator(first, convertedPredPowerInFarm.Keys.Last(), TimeResolution.Hour); skillCalculator.Add(enumer, convertedPredPowerInFarm, convertedObsPowerInFarm, fmd.OffsetHoursAhead); } } } } // 2.1 Print hourly obs series to file for end-user debugging and verification: if(useFixedHours) PrintObsPowerInFarmToDebugFile(obsPowerInFarm); return results; }