public void insertSytheticYear(string collectionName, SyntheticYear synthYear) { var collection = db.GetCollection <SyntheticYear>(collectionName); var f = new Task(() => { collection.InsertOneAsync(synthYear); }); f.Start(); }
private void holeFiller(List <Hole> allHoles, ref SyntheticYear synthYear) { DateTime start = new DateTime(); DateTime end = new DateTime(); DateTime current = new DateTime(); foreach (Hole h in allHoles) { TimeSpan holesize = h.getHoleEnd() - h.getHoleStart(); if (holesize.Hours < 5) { //check for start end year try { if (h.getHoleStart().Month == 1 && h.getHoleStart().Day == 1 && h.getHoleStart().Hour == 0) { //hole at first hour of year start = new DateTime(h.getHoleStart().Year, 12, 31, 23, 0, 0); } else { start = h.getHoleStart().AddHours(-1); } if (h.getHoleEnd().Month == 12 && h.getHoleEnd().Day == 31 && h.getHoleEnd().Hour == 23) { //hole at final hour of year end = new DateTime(h.getHoleStart().Year, 1, 1, 0, 0, 0); } else { end = h.getHoleEnd().AddHours(1); } //interpolation double v1 = synthYear.variables.Find(x => x.name == h.vcode).records.Find(r => r.time == start).value; double v2 = synthYear.variables.Find(x => x.name == h.vcode).records.Find(r => r.time == end).value; double range = v2 - v1; double inc = range / (holesize.Hours + 2); for (int i = 1; i <= holesize.Hours + 1; i++) { current = start.AddHours(i); var tofill = synthYear.variables.Find(x => x.name == h.vcode).records.Find(r => r.time == current); tofill.value = v1 + (i * inc); } } catch (Exception e) { this.addLineToLogFile("WARN: " + h.vcode + " holefilling error between " + h.getHoleStart().ToString() + " and " + h.getHoleEnd().ToString()); } } else { //select from neighour hood of days in week } } }
public static void convertSyntheticYear(ref SyntheticYear yearToCopy) { //input synth year is in UTC time foreach (CollectionMongo vcoll in yearToCopy.variables) { foreach (RecordMongo r in vcoll.records) { r.time = r.time.ToLocalTime(); } } }
//this is for wea to epw public EPWWriter(SyntheticYear locdata, string d) { this.synthYear = locdata; this.directory = d; this.city = this.synthYear.name; this.country = "Colombia";// StringTools.stripExt(new string(this.ld.where)); Directory.CreateDirectory(this.directory + "\\epw"); this.epw = new StreamWriter(this.directory + "\\epw\\" + city + ".epw"); writeEPW(); this.epw.Close(); }
public async Task makeSynthYear(StationGroup sg, string method) { List <IMongoCollection <RecordMongo> > stationData = new List <IMongoCollection <RecordMongo> >(); try { var stationCollNames = getStationsColNames(sg); await index60Minutes(stationCollNames); //ignore 10min collections stationData = getTheStationData(stationCollNames.FindAll(s => s.Contains("60"))); this.addLineToLogFile("INFO: found ref data for " + sg.name + " synth year"); } catch { this.addLineToLogFile("WARN: no ref data found for " + sg.name + " synth year"); } SyntheticYear synthYear = new SyntheticYear(); synthYear.name = sg.name; try { await getDaysForVariables(synthYear, stationData, method); this.addLineToLogFile("INFO: calculated data for " + sg.name + " synth year"); } catch { this.addLineToLogFile("WARN: error in calculating values for " + sg.name + " synth year"); } try { holeFiller(holeFinder(ref synthYear), ref synthYear); this.addLineToLogFile("INFO: hole filling succeeded for " + sg.name + " synth year"); } catch { this.addLineToLogFile("WARN: hole filling failed for " + sg.name + " synth year"); } try { insertSytheticYear(sg.name + "_" + method, synthYear); this.addLineToLogFile("INFO: " + sg.name + " synth year was stored in DB"); } catch { this.addLineToLogFile("WARN: " + sg.name + " synth year was not stored in DB"); } }
public async Task fixCity(string city, string method, NeededData cityNeeds, List <string> stationCollNames) { List <IMongoCollection <RecordMongo> > stationData = new List <IMongoCollection <RecordMongo> >(); try { //ignore 10min collections stationData = getTheStationData(stationCollNames.FindAll(s => s.Contains("_60"))); this.addLineToLogFile("INFO: found ref data for " + city + " synth year"); } catch { this.addLineToLogFile("WARN: no ref data found for " + city + " synth year"); } //read the synthyear for this city var collection = db.GetCollection <SyntheticYear>(city + "_medianHour"); List <SyntheticYear> synthYear = collection.Find(FilterDefinition <SyntheticYear> .Empty).ToList(); SyntheticYear sy = synthYear[0]; SyntheticYear.convertSyntheticYear(ref sy); try { await getDaysForSelectedVariables(sy, stationData, method, cityNeeds.reqVariables); this.addLineToLogFile("INFO: calculated data for " + city + " synth year"); } catch { this.addLineToLogFile("WARN: error in calculating values for " + city + " synth year"); } try { holeFiller(holeFinder(ref sy), ref sy); this.addLineToLogFile("INFO: hole filling succeeded for " + city + " synth year"); } catch { this.addLineToLogFile("WARN: hole filling failed for " + city + " synth year"); } try { insertSytheticYear(city + "_" + method + "regionFix", sy); this.addLineToLogFile("INFO: " + city + " synth year was stored in DB"); } catch { this.addLineToLogFile("WARN: " + city + " synth year was not stored in DB"); } }
private async Task getDaysForVariables(SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData, string method) { if (method == "cdfDay") { await Task.WhenAll(synthYear.variables.Select(c => selectDayOfYearCDF(c.name, synthYear, stationData))); this.addLineToLogFile("INFO: got CDF daily values for " + synthYear.name + " synth year"); } else { await Task.WhenAll(synthYear.variables.Select(c => generateHour(c.name, synthYear, stationData, method))); this.addLineToLogFile("INFO: calculated hourly values for " + synthYear.name + " synth year"); } }
private List <Hole> holeFinder(ref SyntheticYear synthYear) { //fix night radiaiton var rs = synthYear.variables.Find(x => x.name == "RS"); nightRadiation(ref rs); List <Hole> allHoles = new List <Hole>(); foreach (CollectionMongo c in synthYear.variables) { bool newhole = true; Hole h = new Hole(); DateTime prevDT = new DateTime(); foreach (RecordMongo rm in c.records) { if (rm.value == -999.9) { if (newhole) { //found first of new hole h = new Hole(); h.vcode = c.name; h.setHoleStart(rm.time); newhole = false; } } if (rm.value != -999.9 && !newhole) { //end of hole h.setHoleEnd(prevDT); newhole = true; allHoles.Add(h); } prevDT = rm.time; } //hole at end of year if (!newhole) { h.setHoleEnd(prevDT); newhole = true; allHoles.Add(h); } } return(allHoles); }
private async Task getDaysForSelectedVariables(SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData, string method, List <string> fields) { if (method == "cdfDay") { List <Task> tasks = new List <Task>(); foreach (string field in fields) { tasks.Add(selectDayOfYearCDF(field, synthYear, stationData)); } await Task.WhenAll(tasks); this.addLineToLogFile("INFO: got CDF daily values for " + synthYear.name + " synth year"); } else { //List<Task> tasks = new List<Task>(); foreach (string field in fields) { await generateHour(field, synthYear, stationData, method); } //await Task.WhenAll(tasks); this.addLineToLogFile("INFO: calculating hourly values for " + synthYear.name + " synth year"); } }
private async Task selectDayOfYearCDF(string vcode, SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData) { var v = synthYear.variables.Find(x => x.name == vcode); var builder = Builders <RecordMongo> .Filter; string[] pieces; VariableMeta vm; List <int> missingDays = new List <int>(); List <List <RecordMongo> > possDayValues; //find collections with current variable List <IMongoCollection <RecordMongo> > sourceStationData = new List <IMongoCollection <RecordMongo> >(); foreach (IMongoCollection <RecordMongo> sd in stationData) { pieces = sd.CollectionNamespace.CollectionName.Split('_'); if (pieces[4] == vcode) { sourceStationData.Add(sd); } } for (int doy = 1; doy < 366; doy++) { //each day will have several canadidate days sourced from each collection of same variable possDayValues = new List <List <RecordMongo> >(); foreach (IMongoCollection <RecordMongo> sd in sourceStationData) { pieces = sd.CollectionNamespace.CollectionName.Split('_'); string source = pieces[2]; if (source.Contains("NOAA")) { source = "NOAA"; } else { source = "IDEAM"; } vm = AnnualSummary.getVariableMetaFromDB(vcode, source, db); var project = BsonDocument.Parse( "{value: '$value',time:'$time',dayOfYear: {$dayOfYear: '$time'},year: {$year: '$time'}}"); try { //.Match(BsonDocument.Parse("{'dayOfYear' : {$eq : " + doy.ToString() + "}}")) var aggregationDocument = sd.Aggregate() .Unwind("value") .Project(project) .Match(BsonDocument.Parse("{$and:[" + "{'dayOfYear' : {$eq : " + doy.ToString() + "}}" + ",{'value':{$lte:" + vm.max.ToString() + " }}" + ",{'value':{$gte:" + vm.min.ToString() + "}}]}")) .ToList(); IEnumerable <IGrouping <int, BsonDocument> > query = aggregationDocument.GroupBy( doc => doc.GetValue("year").ToInt32(), doc => doc); foreach (IGrouping <int, BsonDocument> yearDayGroup in query) { var year = yearDayGroup.Key; var hours = yearDayGroup.Count(); //one group per day per year count should be 24 //but many noaa data are sometimes day time only 6-6 12 readings if (hours >= 12) { List <RecordMongo> dayValues = new List <RecordMongo>(); foreach (BsonDocument name in yearDayGroup) { RecordMongo rm = new RecordMongo(); double value = name.GetValue("value").ToDouble(); //check nub and HRs are in the right range if (vcode == "HR" && value <= 1) { value = value * 100; } if (vcode == "NUB") { //noaa's cloud is oktas if (value == 9) { value = 10; } else { value = (int)(value / 8.0 * 10); } } rm.value = value; rm.time = name.GetValue("time").ToLocalTime(); dayValues.Add(rm); } possDayValues.Add(dayValues); } } } catch (Exception e) { this.addLineToLogFile("WARN: " + synthYear.name + vcode + " error finding cdf day at day of year: " + doy); } } if (possDayValues.Count > 0) { List <RecordMongo> dayToInsert = typicalDay(possDayValues, vcode); addValuesToSynthYear(dayToInsert, ref v, doy, vcode); } else { //no possible days found empty day missingDays.Add(doy); } } if (missingDays.Count > 0) { fillMissingDays(missingDays); } //fill missing days }
private async Task generateHour2(string vcode, SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData, string meanMedian) { var v = synthYear.variables.Find(x => x.name == vcode); var builder = Builders <RecordMongo> .Filter; string[] pieces; VariableMeta vm; int hourofsyntheticyear = 0; DateTime universal = new DateTime(); //find collections with current variable List <IMongoCollection <RecordMongo> > sourceStationData = new List <IMongoCollection <RecordMongo> >(); foreach (IMongoCollection <RecordMongo> sd in stationData) { pieces = sd.CollectionNamespace.CollectionName.Split('_'); if (pieces[4] == vcode) { sourceStationData.Add(sd); } } foreach (RecordMongo r in v.records) { //synth year is local time universal = r.time.ToUniversalTime(); int h = universal.Hour; int doy = universal.DayOfYear; hourofsyntheticyear++; int foundValues = 0; List <double> valuesForHour = new List <double>(); foreach (IMongoCollection <RecordMongo> sd in sourceStationData) { pieces = sd.CollectionNamespace.CollectionName.Split('_'); string source = pieces[2]; if (source.Contains("NOAA")) { source = "NOAA"; } else { source = "IDEAM"; } vm = AnnualSummary.getVariableMetaFromDB(vcode, source, db); var project = BsonDocument.Parse( "{value: '$value',time:'$time',dayOfYear: {$dayOfYear: '$time'},hour: {$hour: '$time'}}"); try { var aggregationDocument = sd.Aggregate() .Unwind("value") .Project(project) .Match(BsonDocument.Parse("{$and:[" + "{'dayOfYear' : {$eq : " + doy.ToString() + "}}" + ",{'hour' : {$eq : " + h.ToString() + "}}" + ",{'value':{$lte:" + vm.max.ToString() + " }}" + ",{'value':{$gte:" + vm.min.ToString() + "}}]}")) .ToList(); IEnumerable <IGrouping <int, BsonDocument> > query = aggregationDocument.GroupBy( doc => doc.GetValue("dayOfYear").ToInt32(), doc => doc); foreach (IGrouping <int, BsonDocument> hourValsGroup in query) { foreach (BsonDocument name in hourValsGroup) { double value = name.GetValue("value").ToDouble(); foundValues++; //check nub and HRs are in the right range if (vcode == "HR" && value <= 1) { value = value * 100; } if (vcode == "NUB") { //noaa's cloud is oktas if (value == 9) { value = 10; } else { value = (int)(value / 8.0 * 10); } } valuesForHour.Add(value); } } } catch (Exception e) { this.addLineToLogFile("WARN: " + synthYear.name + vcode + " error finding hourly values at day of year: " + doy + " ,hour : " + h); } } if (foundValues != 0) { if (meanMedian == "meanHour") { r.value = Accord.Statistics.Measures.Mean(valuesForHour.ToArray()); } if (meanMedian == "medianHour") { r.value = Accord.Statistics.Measures.Median(valuesForHour.ToArray()); } if (meanMedian == "randomHour") { int total = valuesForHour.Count; Random rand = new Random(); r.value = valuesForHour[rand.Next(0, total)]; } } } }
private async Task generateHour(string vcode, SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData, string meanMedian) { var v = synthYear.variables.Find(x => x.name == vcode); var builder = Builders <RecordMongo> .Filter; string[] pieces; VariableMeta vm; int hourofsyntheticyear = 0; DateTime local = new DateTime(); DateTime universal = new DateTime(); //find collections with current variable List <IMongoCollection <RecordMongo> > sourceStationData = new List <IMongoCollection <RecordMongo> >(); foreach (IMongoCollection <RecordMongo> sd in stationData) { pieces = sd.CollectionNamespace.CollectionName.Split('_'); if (pieces[4] == vcode) { sourceStationData.Add(sd); } } foreach (RecordMongo r in v.records) { //this is the time we need to fill //need to filter for month day and hour int m = r.time.Month; int d = r.time.Day; int h = r.time.Hour; hourofsyntheticyear++; double value = 0; int foundValues = 0; List <double> valuesForHour = new List <double>(); foreach (IMongoCollection <RecordMongo> sd in sourceStationData) { //only if the vcode matches pieces = sd.CollectionNamespace.CollectionName.Split('_'); string source = pieces[2]; vm = AnnualSummary.getVariableMetaFromDB(vcode, source, db); int startYr = 0; int endYr = 0; getFirstLastYear(sd, ref startYr, ref endYr); if (startYr == 1) { startYr = 1980; } if (endYr == 1) { startYr = 2018; } for (int y = startYr; y < endYr; y++) { local = new DateTime(y, m, d, h, 0, 0); universal = local.ToUniversalTime(); var filter = builder.Eq("time", universal) & builder.Gte("value", vm.min) & builder.Lte("value", vm.max); //some collections have duplicate timestamps! using (IAsyncCursor <RecordMongo> cursor = await sd.FindAsync(filter)) { while (await cursor.MoveNextAsync()) { IEnumerable <RecordMongo> documents = cursor.Current; //insert into the station collection foreach (RecordMongo sdrm in documents) { value = sdrm.value; if (vcode == "HR" && sdrm.value <= 1) { value = value * 100; } if (vcode == "NUB") { if (value == 9) { value = 10; } value = (int)(value / 8.0 * 10); } valuesForHour.Add(value); foundValues++; } } } } } if (foundValues != 0) { if (meanMedian == "meanHour") { r.value = Accord.Statistics.Measures.Mean(valuesForHour.ToArray()); } if (meanMedian == "medianHour") { r.value = Accord.Statistics.Measures.Median(valuesForHour.ToArray()); } if (meanMedian == "randomHour") { int total = valuesForHour.Count; Random rand = new Random(); r.value = valuesForHour[rand.Next(0, total)]; } } } }