public async Task <IActionResult> HistoricalYield(string source, IFormFile file) { Message msg = null; try { if (file != null && file.Length > 0) { // Save a copy in the web site await file.CopyToAsync(new FileStream(importPath + DateTime.Now.ToString("yyyyMMddHHmmss") + "-weatherstation-historical-yield-ws-" + file.FileName, FileMode.Create)); // Read the file StreamReader reader = new StreamReader(file.OpenReadStream()); string line = string.Empty; int lines = 0, total_lines = 0; List <HistoricalYieldViewImport> raw = new List <HistoricalYieldViewImport>(); string[] values = null; // Read the file while (!reader.EndOfStream) { line = await reader.ReadLineAsync(); if (!string.IsNullOrEmpty(line)) { lines += 1; total_lines += 1; // Don't reade the headers if (lines != 1) { try { // Fixed the values to view import historical climate // The first two columns are the year and month values = line.Split(','); raw.Add(new HistoricalYieldViewImport() { weather_station = values[0], soil = values[1], cultivar = values[2], start = DateTime.SpecifyKind(DateTime.Parse(values[3]), DateTimeKind.Utc), end = DateTime.SpecifyKind(DateTime.Parse(values[4]), DateTimeKind.Utc), measure = (MeasureYield)Enum.Parse(typeof(MeasureYield), values[5]), median = double.Parse(values[6]), avg = double.Parse(values[7]), min = double.Parse(values[8]), max = double.Parse(values[9]), quar_1 = double.Parse(values[10]), quar_2 = double.Parse(values[11]), quar_3 = double.Parse(values[12]), conf_lower = double.Parse(values[13]), conf_upper = double.Parse(values[14]), sd = double.Parse(values[15]), perc_5 = double.Parse(values[16]), perc_95 = double.Parse(values[17]), coef_var = double.Parse(values[18]) }); } catch (Exception ex) { lines -= 1; Console.WriteLine(ex); } } } } // Import to the database HistoricalYield hy_new; YieldCrop yc_entity; List <YieldCrop> yc_entities; List <YieldData> yd_entities; // En esta sección se crean los nuevos registros de rendimientos para las estaciones climatologicas. foreach (var ws in raw.Select(p => new { p.weather_station, p.cultivar, p.soil }).Distinct()) { hy_new = new HistoricalYield() { source = getId(source), weather_station = getId(ws.weather_station), soil = getId(ws.soil), cultivar = getId(ws.cultivar) }; var yield_crop = raw.Where(p => p.weather_station == ws.weather_station && p.soil == ws.soil && p.cultivar == ws.cultivar); yc_entities = new List <YieldCrop>(); foreach (var yc in yield_crop) { yc_entity = new YieldCrop() { start = yc.start, end = yc.end }; var yield_data = yield_crop.Where(p => p.cultivar == yc.cultivar && p.soil == yc.soil && p.start == yc.start && p.end == yc.end); int count_yd = yield_data.Count(); yd_entities = new List <YieldData>(); foreach (var yd in yield_data) { yd_entities.Add(new YieldData() { measure = yd.measure, median = yd.median, avg = yd.avg, min = yd.min, max = yd.max, quar_1 = yd.quar_1, quar_2 = yd.quar_2, quar_3 = yd.quar_3, conf_lower = yd.conf_lower, conf_upper = yd.conf_upper, sd = yd.sd, perc_5 = yd.perc_5, perc_95 = yd.perc_95, coef_var = yd.coef_var }); } yc_entity.data = yd_entities; yc_entities.Add(yc_entity); } hy_new.yield = yc_entities; await db.historicalYield.insertAsync(hy_new); } msg = new Message() { content = "Historical Yield WS. The file was imported correctly. Records imported: " + (lines - 1).ToString() + " from: " + (total_lines - 1).ToString() + " rows", type = MessageType.successful }; await writeEventAsync(msg.content, LogEvent.cre, new List <LogEntity>() { LogEntity.lc_weather_station, LogEntity.hs_historical_yield }); } else { msg = new Message() { content = "Historical Yield WS. An error occurred with the file imported", type = MessageType.error }; await writeEventAsync(msg.content, LogEvent.err, new List <LogEntity>() { LogEntity.lc_weather_station }); } } catch (Exception ex) { await writeExceptionAsync(ex); msg = new Message() { content = "Historical Yield WS. An error occurred in the system, contact the administrator", type = MessageType.error }; } // List climate variables await generateListMeasuresAndSourceAsync(); ViewBag.message = msg; return(View("HistoricalYield")); }
public async Task <bool> importForecastAsync(string path, double cf) { StreamReader file; string line; // Load the avaliable cpt configuration by state List <ClimateConfiguration> climate_conf = new List <ClimateConfiguration>(); var states = await db.state.listEnableAsync(); foreach (var s in states) { climate_conf.Add(new ClimateConfiguration() { state = s.id, conf = s.conf.ToList().Where(p => p.track.enable) }); } // Create a forecast var forecast = await db.forecast.insertAsync(new Data.Models.Forecast() { start = DateTime.Now, end = DateTime.Now, confidence = cf, climate_conf = climate_conf }); Console.WriteLine("Created forecast " + forecast.id.ToString()); // Load probabilities Console.WriteLine("Getting probabilities and performance"); Console.WriteLine(path + Program.settings.In_PATH_FS_CLIMATE + Path.DirectorySeparatorChar + Program.settings.In_PATH_FS_PROBABILITIES); var f_probabilities = Directory.EnumerateFiles(path + Program.settings.In_PATH_FS_CLIMATE + Path.DirectorySeparatorChar + Program.settings.In_PATH_FS_PROBABILITIES); // Get the probabilities file List <ImportProbability> probabilities = new List <ImportProbability>(); string fp = f_probabilities.SingleOrDefault(p => p.Contains(Program.settings.In_PATH_FS_FILE_PROBABILITY)); if (!string.IsNullOrEmpty(fp)) { Console.WriteLine("Processing: probabilities"); Console.WriteLine(fp); // Reading the file using (file = File.OpenText(fp)) { int count = 0; while ((line = file.ReadLine()) != null) { // Omitted the file's header if (count != 0 && !string.IsNullOrEmpty(line)) { // Get the probabilities from the file in a temp memmory var fields = line.Split(Program.settings.splitted); probabilities.Add(new ImportProbability() { year = int.Parse(fields[0]), month = int.Parse(fields[1]), ws = fields[2], below = double.Parse(fields[3]), normal = double.Parse(fields[4]), above = double.Parse(fields[5]) }); } count += 1; } } } else { Console.WriteLine("Probabilities not found"); } // Get the performance metrics file List <ImportPerformance> performances = new List <ImportPerformance>(); string fpe = f_probabilities.SingleOrDefault(p => p.Contains(Program.settings.In_PATH_FS_FILE_PERFORMANCE)); if (!string.IsNullOrEmpty(fpe)) { Console.WriteLine("Processing: performance"); Console.WriteLine(fpe); // Reading the file using (file = File.OpenText(fpe)) { int count = 0; while ((line = file.ReadLine()) != null) { // Omitted the file's header if (count != 0 && !string.IsNullOrEmpty(line)) { // Get the probabilities from the file in a temp memmory var fields = line.Split(Program.settings.splitted); performances.Add(new ImportPerformance() { year = int.Parse(fields[0]), month = int.Parse(fields[1]), ws = fields[2], pearson = double.Parse(fields[3]), kendall = double.Parse(fields[4]), goodness = double.Parse(fields[5]) }); } count += 1; } } } else { Console.WriteLine("Performances not found"); } // Create the records of the probabilities in the database Console.WriteLine("Saving the probabilities and metrics in the database"); foreach (var ws in probabilities.Select(p => p.ws).Distinct()) { // Casting the metrics List <PerformanceMetric> metrics = new List <PerformanceMetric>(); foreach (var m in Enum.GetValues(typeof(MeasurePerformance)).Cast <MeasurePerformance>()) { if (m == MeasurePerformance.goodness) { metrics.AddRange(performances.Where(p => p.ws.Equals(ws)).Select(p => new PerformanceMetric() { year = p.year, month = p.month, name = m, value = p.goodness }).ToList()); } else if (m == MeasurePerformance.kendall) { metrics.AddRange(performances.Where(p => p.ws.Equals(ws)).Select(p => new PerformanceMetric() { year = p.year, month = p.month, name = m, value = p.kendall }).ToList()); } else if (m == MeasurePerformance.pearson) { metrics.AddRange(performances.Where(p => p.ws.Equals(ws)).Select(p => new PerformanceMetric() { year = p.year, month = p.month, name = m, value = p.pearson }).ToList()); } } // Saving in the database await db.forecastClimate.insertAsync(new ForecastClimate() { forecast = forecast.id, weather_station = ForecastDB.parseId(ws), data = probabilities.Where(p => p.ws.Equals(ws)).Select(p => new ProbabilityClimate() { year = p.year, month = p.month, probabilities = new List <Probability>() { new Probability() { measure = MeasureClimatic.prec, lower = p.below, normal = p.normal, upper = p.above } } }).OrderBy(p => p.year).ThenBy(p => p.month).ToList(), performance = metrics.OrderBy(p => p.year).ThenBy(p => p.month).ToList() }); } // Load scenarios Console.WriteLine("Getting scenarios"); // Get folder of the scenarios List <ImportScenario> scenarios = new List <ImportScenario>(); Console.WriteLine("Searching scenarios"); Console.WriteLine(path + Program.settings.In_PATH_FS_CLIMATE + Path.DirectorySeparatorChar + Program.settings.In_PATH_FS_SCENARIOS); // Get a list of files of the scenarios var d_f_scenario = Directory.EnumerateFiles(path + Program.settings.In_PATH_FS_CLIMATE + Path.DirectorySeparatorChar + Program.settings.In_PATH_FS_SCENARIOS).Where(p => !p.Contains("scenario")).OrderBy(p => p); // This cicle goes through of the scenarios (max, min, avg) foreach (var s in Enum.GetNames(typeof(ScenarioName))) { // This cicle goes through of the measure (t_max, t_min, ) foreach (var me in Enum.GetNames(typeof(MeasureClimatic))) { var f_scenarios = d_f_scenario.Where(p => p.Contains(me + "_" + s)); if (f_scenarios.Count() == 0) { Console.WriteLine("File not found. scenario: " + s + " measure: " + me); } foreach (var fs in f_scenarios) { Console.WriteLine("Getting scenario: " + s + " measure: " + me); Console.WriteLine(fs); var ws_fs = fs.Split(Path.DirectorySeparatorChar)[fs.Split(Path.DirectorySeparatorChar).Length - 1].Substring(0, 24); Console.WriteLine(ws_fs); // Reading the file using (file = File.OpenText(fs)) { int count = 0; while ((line = file.ReadLine()) != null) { // Omitted the file's header if (count != 0 && !string.IsNullOrEmpty(line)) { // Get the data from the file in a temp memmory var fields = line.Split(Program.settings.splitted); scenarios.Add(new ImportScenario() { year = int.Parse(fields[0]), month = int.Parse(fields[1]), ws = ws_fs, scenario = s, measure = me, value = double.Parse(fields[2]) }); } count += 1; } } } } } // Create the records of the scenarios in the database Console.WriteLine("Saving the scenarios in the database"); var header_scenario = scenarios.Select(p => new { p.ws, p.year, p.scenario, }).Distinct().OrderBy(p => p.ws).ThenBy(p => p.year); foreach (var data in header_scenario) { var data_temp = scenarios.Where(p => p.ws == data.ws && p.scenario == data.scenario && p.year == data.year). OrderBy(p => p.measure).ThenBy(p => p.month); List <MonthlyDataStation> monthly_data = new List <MonthlyDataStation>(); foreach (var month in data_temp.Select(p => p.month).Distinct()) { monthly_data.Add(new MonthlyDataStation() { month = month, data = data_temp.Where(p => p.month == month).Select(p2 => new ClimaticData() { measure = (MeasureClimatic)Enum.Parse(typeof(MeasureClimatic), p2.measure, true), value = p2.value }).ToList() }); } await db.forecastScenario.insertAsync(new ForecastScenario() { forecast = forecast.id, weather_station = ForecastDB.parseId(data.ws), name = (ScenarioName)Enum.Parse(typeof(ScenarioName), data.scenario, true), year = data.year, monthly_data = monthly_data }); } // Load yield data Console.WriteLine("Copying raster"); Console.WriteLine(path + Program.settings.In_PATH_FS_CLIMATE + Path.DirectorySeparatorChar + Program.settings.In_PATH_FS_RASTER_SOURCE); if (Directory.Exists(Program.settings.In_PATH_FS_RASTER_DESTINATION)) { string raster_d = Program.settings.In_PATH_FS_RASTER_DESTINATION + Path.DirectorySeparatorChar + forecast.id.ToString(); DirectoryHelper.DirectoryCopy(path + Program.settings.In_PATH_FS_CLIMATE + Path.DirectorySeparatorChar + Program.settings.In_PATH_FS_RASTER_SOURCE, raster_d, true); } else { Console.WriteLine("Folder to save the raster files doesn't exist: " + Program.settings.In_PATH_FS_RASTER_DESTINATION); } // Load yield data Console.WriteLine("Getting yield"); Console.WriteLine(path + Program.settings.In_PATH_FS_YIELD); // Get folder of the scenarios var d_crops = Directory.EnumerateDirectories(path + Program.settings.In_PATH_FS_YIELD); List <ImportYield> yields = new List <ImportYield>(); foreach (var dc in d_crops) { Console.WriteLine("Crop " + dc); var d_f_yield = Directory.EnumerateFiles(dc); foreach (var f_yield in d_f_yield.Where(p => p.EndsWith(".csv"))) { Console.WriteLine("Working in " + f_yield); using (file = File.OpenText(f_yield)) { int count = 0; while ((line = file.ReadLine()) != null) { // Omitted the file's header if (count != 0 && !string.IsNullOrEmpty(line)) { // Get the data from the file in a temp memmory var fields = line.Split(Program.settings.splitted); yields.Add(new ImportYield() { weather_station = fields[0], soil = fields[1], cultivar = fields[2], start = Program.settings.Add_Day ? DateTime.Parse(fields[3]).AddDays(1) : DateTime.Parse(fields[3]), end = Program.settings.Add_Day ? DateTime.Parse(fields[4]).AddDays(1) : DateTime.Parse(fields[4]), measure = fields[5], avg = double.Parse(fields[6].ToLower().Equals("nan") ? "0" : fields[6]), median = double.Parse(fields[7].ToLower().Equals("nan") ? "0" : fields[7]), min = double.Parse(fields[8].ToLower().Equals("nan") ? "0" : fields[8]), max = double.Parse(fields[9].ToLower().Equals("nan") ? "0" : fields[9]), quar_1 = double.Parse(fields[10].ToLower().Equals("nan") ? "0" : fields[10]), quar_2 = double.Parse(fields[11].ToLower().Equals("nan") ? "0" : fields[11]), quar_3 = double.Parse(fields[12].ToLower().Equals("nan") ? "0" : fields[12]), conf_lower = double.Parse(fields[13].ToLower().Equals("nan") ? "0" : fields[13]), conf_upper = double.Parse(fields[14].ToLower().Equals("nan") ? "0" : fields[14]), sd = double.Parse(fields[15].ToLower().Equals("nan") ? "0" : fields[15]), perc_5 = double.Parse(fields[16].ToLower().Equals("nan") ? "0" : fields[16]), perc_95 = double.Parse(fields[17].ToLower().Equals("nan") ? "0" : fields[17]), coef_var = double.Parse(fields[18].ToLower().Equals("nan") ? "0" : fields[18]) }); } count += 1; } } } } // Create the records of the yield in the database Console.WriteLine("Saving the yield in the database"); ForecastYield fy_new; YieldCrop yc_entity; List <YieldCrop> yc_entities; List <YieldData> yd_entities; var ws_list = yields.Select(p => new { p.weather_station, p.cultivar, p.soil }).Distinct(); foreach (var ws in ws_list) { Console.WriteLine("Working in ws: " + ws.weather_station + " soil: " + ws.soil + " cultivar: " + ws.cultivar); fy_new = new ForecastYield() { forecast = forecast.id, weather_station = ForecastDB.parseId(ws.weather_station), cultivar = ForecastDB.parseId(ws.cultivar), soil = ForecastDB.parseId(ws.soil) }; var yield_crop = yields.Where(p => p.weather_station == ws.weather_station && p.soil == ws.soil && p.cultivar == ws.cultivar); yc_entities = new List <YieldCrop>(); var dates_list = yield_crop.Select(p => new { start = p.start, end = p.end }).Distinct(); int count_yc = yield_crop.Count(); foreach (var dc in dates_list) { yc_entity = new YieldCrop() { start = dc.start, end = dc.end }; var yield_data = yield_crop.Where(p => p.start == dc.start && p.end == dc.end); int count_yd = yield_data.Count(); yd_entities = new List <YieldData>(); foreach (var yd in yield_data) { yd_entities.Add(new YieldData() { measure = (MeasureYield)Enum.Parse(typeof(MeasureYield), yd.measure, true), median = yd.median, avg = yd.avg, min = yd.min, max = yd.max, quar_1 = yd.quar_1, quar_2 = yd.quar_2, quar_3 = yd.quar_3, conf_lower = yd.conf_lower, conf_upper = yd.conf_upper, sd = yd.sd, perc_5 = yd.perc_5, perc_95 = yd.perc_95, coef_var = yd.coef_var }); } yc_entity.data = yd_entities; yc_entities.Add(yc_entity); } fy_new.yield = yc_entities; await db.forecastYield.insertAsync(fy_new); } Console.WriteLine("Forecast imported"); return(true); }