public static void SaveDataSet(MeteoDataSet dataSet, string path, LabelizationMode labelizationMode)
{
var groupSize = dataSet.TrainData[0].Inputs.Count;
string directoryPart = $@"{path}PreparedData\{groupSize}\Labelization_{labelizationMode.ToString()}\";
string filePart = $"{dataSet.Season.ToString()}_{DS_TYPE}.csv";
var pathToSave = directoryPart + filePart;
Directory.CreateDirectory(directoryPart);
using (StreamWriter sw = new StreamWriter(pathToSave.Replace(DS_TYPE, "Train")))
{
sw.WriteLine(MeteoData.GetHeader(groupSize));
foreach (var gr in dataSet.TrainData)
{
sw.WriteLine(gr.ToString(labelizationMode));
}
}
using (StreamWriter sw = new StreamWriter(pathToSave.Replace(DS_TYPE, "Test")))
{
sw.WriteLine(MeteoData.GetHeader(groupSize));
foreach (var gr in dataSet.TestData)
{
sw.WriteLine(gr.ToString(labelizationMode));
}
}
}
private void AssignItemToSeason(MeteoData keyItem)
{
DateTime springStart = new DateTime(2000, 3, 21);
DateTime summerStart = new DateTime(2000, 6, 22);
DateTime autumnStart = new DateTime(2000, 9, 23);
DateTime winterStart = new DateTime(2000, 12, 22);
springStart = new DateTime(keyItem.Date.Year, springStart.Month, springStart.Day);
summerStart = new DateTime(keyItem.Date.Year, summerStart.Month, summerStart.Day);
autumnStart = new DateTime(keyItem.Date.Year, autumnStart.Month, autumnStart.Day);
winterStart = new DateTime(keyItem.Date.Year, winterStart.Month, winterStart.Day);
if (keyItem.Date >= springStart && keyItem.Date < summerStart)
{
springItems.Add(keyItem);
}
else if (keyItem.Date >= summerStart && keyItem.Date < autumnStart)
{
summerItems.Add(keyItem);
}
else if (keyItem.Date >= autumnStart && keyItem.Date < winterStart)
{
autumnItems.Add(keyItem);
}
else if (keyItem.Date >= winterStart || keyItem.Date < springStart)
{
winterItems.Add(keyItem);
}
}
public void LabelizeRecord(MeteoData item, Season season)
{
item.WindDirectionL = GetWindDirectionLabel(item.WindDirection);
item.WindSpeedL = GetWindSpeedLabel(item.WindSpeed);
item.CloudyAvgL = GetCloudyLabel(item.CloudyAvg);
item.Rain12hL = GetRainLabel(item.Rain12h);
item.TempAvgL = GetTempLabel(item.TempAvg, season);
item.TempMinL = GetTempMinLabel(item.TempMin, season);
item.TempMaxL = GetTempMaxLabel(item.TempMax, season);
item.PressureStationLevelL = GetPreasureLabel(item.PressureStationLevel);
}
private static bool IsRecordValid(MeteoDataRecord group)
{
MeteoData prevInput = group.Inputs[0];
for (int i = 1; i < group.Inputs.Count; i++)
{
if (prevInput.Date.AddDays(1) != group.Inputs[i].Date)
{
return(false);
}
prevInput = group.Inputs[i];
}
if (prevInput.Date.AddDays(1) != group.Output.Date)
{
return(false);
}
return(true);
}
public MeteoData(MeteoData rec)
{
this.Date = rec.Date;
this.TempMax = rec.TempMax;
this.TempMin = rec.TempMin;
this.TempAvg = rec.TempAvg;
this.DevPointTempAvg = rec.DevPointTempAvg;
this.GroundTempMin = rec.GroundTempMin;
this.WettedThermMaxTemp = rec.WettedThermMaxTemp;
this.TempMaxAnomaly = rec.TempMaxAnomaly;
this.TempMinAnomaly = rec.TempMinAnomaly;
this.TempMAvgAnomaly = rec.TempMAvgAnomaly;
this.CloudyAvg = rec.CloudyAvg;
this.WindSpeed = rec.WindSpeed;
this.WindDirection = rec.WindDirection;
this.PressureSeeLevel = rec.PressureSeeLevel;
this.PressureStationLevel = rec.PressureStationLevel;
this.Rain12h = rec.Rain12h;
this.AvgVisibility = rec.AvgVisibility;
//LabelizeInput(s);
//PrepareOutput(s);
}
private void ReadFromHtml(int year)
{
int readRecords = 0;
foreach (var file in Directory.GetFiles($@"{path}{year}\OnlyTables"))
{
if (file.EndsWith(".html"))
{
HtmlDocument doc = new HtmlDocument();
doc.Load(file);
int rowCounter = 0;
var allRows = doc.DocumentNode.SelectNodes("//table[@id='tablepl']//tr");
foreach (HtmlNode row in allRows)
{
if (rowCounter++ < 2) //skip table headers
{
continue;
}
MeteoData meteoRecord = new MeteoData();
bool save = true;
int cellIdx = 1;
foreach (var cell in row.SelectNodes(".//td"))
{
if (!KEY_PROPERTIES.Contains(cellIdx))
{
cellIdx++;
continue;
}
string cellValue;
if (cellIdx == 14)
{
cellValue = MapWindProperty(cell);
}
else if (cellIdx > 20)
{
cellValue = MapImgProperty(cell);
}
else
{
cellValue = cell.FirstChild.InnerText.Trim();
}
if (!StoreCellValue(cellIdx, meteoRecord, cellValue))
{
save = false;
EmptyRecords++;
break;
}
cellIdx++;
}
if (save && !allItems.Any(r => r.Date == meteoRecord.Date))
{
readRecords++;
allItems.Add(meteoRecord);
}
}
}
}
Console.WriteLine($"Data for {year} read. Records: {readRecords}");
}
private bool StoreCellValue(int cellIdx, MeteoData row, string value)
{
value = value.Replace('.', ',');
if (value == string.Empty || value == "-")
{
value = string.Empty;
}
if (string.IsNullOrEmpty(value))
{
return(false);
}
switch (cellIdx)
{
case 1: row.Date = DateTime.Parse(value); break;
case 3: row.TempMax = Double.Parse(value); break;
case 4: row.TempMin = Double.Parse(value); break;
case 5: row.TempAvg = Double.Parse(value); break;
case 6: row.DevPointTempAvg = Double.Parse(value); break;
case 7: row.GroundTempMin = Double.Parse(value); break;
case 8: row.WettedThermMaxTemp = Double.Parse(value); break;
case 9: row.TempMaxAnomaly = Double.Parse(value); break;
case 10: row.TempMinAnomaly = Double.Parse(value); break;
case 11: row.TempMAvgAnomaly = Double.Parse(value); break;
case 12: row.CloudyAvg = Double.Parse(value); break;
case 13: row.WindSpeed = Double.Parse(value); break;
case 14: row.WindDirection = Double.Parse(value); break;
case 15: row.MaxWindGust = Double.Parse(value); break;
case 16: row.PressureSeeLevel = Double.Parse(value); break;
case 17: row.PressureStationLevel = Double.Parse(value); break;
case 18: row.Rain24h = Double.Parse(value); break;
case 19: row.Rain12h = Double.Parse(value); break;
case 20: row.AvgVisibility = Double.Parse(value); break;
case 21: row.Insolation = Double.Parse(value); break;
case 22: row.IceSheetHeight = Double.Parse(value); break;
case 23: row.Storm = Double.Parse(value) == 1.0; break;
case 24: row.Rain = Double.Parse(value) == 1.0; break;
case 25: row.Snow = Double.Parse(value) == 1.0; break;
case 26: row.Fog = Double.Parse(value) == 1.0; break;
}
return(true);
}
