public void cleanSingle(string collection) { string[] parts = collection.Split('_'); int stationcode = Convert.ToInt32(parts[1]); string vname = parts[4]; if (vname == "PA") { return; } string source = parts[2]; int freq = Convert.ToInt32(parts[5]); VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); string newname = convertNameToClean(collection); //collection for the avergaed data CollectionMongo cm = new CollectionMongo(); cm.name = newname; newCleanData.Add(cm); Task t1 = Task.Run(() => removeRecordsOutsideRange(stationcode, collection, meta, newname)); t1.Wait(); insertMany(cm.records, cm.name); }
public void cleanUp() { List <string> collNames = MongoTools.collectionNames(db); string vname = ""; int stationcode = 0; string source = ""; int freq = 0; foreach (string collection in collNames) { //all station record collections start with an s_ if (collection[0] == 's') { string[] parts = collection.Split('_'); stationcode = Convert.ToInt32(parts[1]); vname = parts[4]; if (vname == "PA") { continue; } source = parts[2]; freq = Convert.ToInt32(parts[5]); VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); if (freq == 60) { string newname = convertNameToClean(collection); CollectionMongo cm = new CollectionMongo(); cm.name = newname; removeRecordsOutsideRange(stationcode, collection, meta, newname); } } } }
public void convertSingleCollection(string collection) { string[] parts = collection.Split('_'); int stationcode = Convert.ToInt32(parts[1]); string vname = parts[4]; string source = parts[2]; int freq = Convert.ToInt32(parts[5]); if (freq == 60) { return; } VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); string newname = convertNameTo60min(collection); //collection for the avergaed data CollectionMongo cm = new CollectionMongo(); cm.name = newname; newAveragedData.Add(cm); Task t1 = Task.Run(() => sortByDateAndAverage(stationcode, collection, meta, newname)); t1.Wait(); insertMany(cm.records, cm.name); }
private async Task cityGroupGraphic(StationGroup cityGroup, List <string> weatherCollections, string filename) { List <City> cities = MapTools.readCities(); var c = cities.Find(x => x.name == cityGroup.name); //start end dates //set the master pane ZedGraphControl zgc = new ZedGraphControl(); MasterPane master = zgc.MasterPane; master.Rect = new RectangleF(0, 0, 2000, 666 * weatherCollections.Count); master.PaneList.Clear(); master.Title.Text = "City: " + c.name + " lat: " + Math.Round(c.location[1], 3) + " lon: " + Math.Round(c.location[0], 3) + " alt: " + (int)(c.elevation);// + "\nDate range: " + startDate.Year + "_" + startDate.Month + " >> " + endDate.Year + "_" + endDate.Month; master.Title.FontSpec = new FontSpec("Arial", 7.0f, Color.Black, false, false, false); master.Margin.All = 5; master.Legend.IsVisible = false; int stationcode = 0; string vname = ""; string source = ""; int freq = 0; foreach (string wc in weatherCollections) { ////create one scatter for each stationvariable if (!wc.Contains("Clean")) { string[] parts = wc.Split('_'); stationcode = Convert.ToInt32(parts[1]); vname = parts[4]; source = parts[2]; freq = Convert.ToInt32(parts[5]); VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); PointPairList pointpair = new PointPairList(); pointpair = await GenerateHourlyData(wc, meta); if (pointpair.Count > 0) { AddChartToMaster(master, pointpair, wc, vname, meta, false); } } } //save graphic // Refigure the axis ranges for the GraphPanes zgc.AxisChange(); // Layout the GraphPanes using a default Pane Layout Bitmap b = new Bitmap(2000, 666 * weatherCollections.Count); using (Graphics g = Graphics.FromImage(b)) { master.SetLayout(g, PaneLayout.SingleColumn); } master.GetImage().Save(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\IDEAM\DailyAnalysis\" + filename + ".jpeg", System.Drawing.Imaging.ImageFormat.Jpeg); }
private async Task graphSingleStation(string collection, string filename) { //set the master pane ZedGraphControl zgc = new ZedGraphControl(); MasterPane master = zgc.MasterPane; master.Rect = new RectangleF(0, 0, 2000, 666); master.PaneList.Clear(); //master.Title.Text = "City: " + c.name + // " lat: " + Math.Round(c.location[1], 3) + " lon: " + Math.Round(c.location[0], 3) + // " alt: " + (int)(c.elevation);// + "\nDate range: " + startDate.Year + "_" + startDate.Month + " >> " + endDate.Year + "_" + endDate.Month; master.Title.FontSpec = new FontSpec("Arial", 7.0f, Color.Black, false, false, false); master.Margin.All = 5; master.Legend.IsVisible = false; int stationcode = 0; string vname = ""; string source = ""; int freq = 0; string[] parts = collection.Split('_'); stationcode = Convert.ToInt32(parts[1]); vname = parts[4]; source = parts[2]; freq = Convert.ToInt32(parts[5]); VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); PointPairList pointpair = new PointPairList(); pointpair = await GenerateHourlyData(collection, meta); if (pointpair.Count > 0) { AddChartToMaster(master, pointpair, collection, vname, meta, false); } //save graphic // Refigure the axis ranges for the GraphPanes zgc.AxisChange(); // Layout the GraphPanes using a default Pane Layout Bitmap b = new Bitmap(2000, 666); using (Graphics g = Graphics.FromImage(b)) { master.SetLayout(g, PaneLayout.SingleColumn); } master.GetImage().Save(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\IDEAM\DailyAnalysisTS_RS\" + filename + ".jpeg", System.Drawing.Imaging.ImageFormat.Jpeg); }
private async Task testDaysGraphic(List <string> weatherCollections, string filename) { //set the master pane ZedGraphControl zgc = new ZedGraphControl(); MasterPane master = zgc.MasterPane; master.Rect = new RectangleF(0, 0, 2000, 666 * weatherCollections.Count); master.PaneList.Clear(); master.Title.FontSpec = new FontSpec("Arial", 7.0f, Color.Black, false, false, false); master.Margin.All = 5; master.Legend.IsVisible = false; int stationcode = 0; string vname = ""; string source = ""; int freq = 0; foreach (string wc in weatherCollections) { ////create one scatter for each stationvariable string[] parts = wc.Split('_'); stationcode = Convert.ToInt32(parts[1]); vname = parts[4]; source = parts[2]; freq = Convert.ToInt32(parts[5]); VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); PointPairList pointpair = new PointPairList(); pointpair = await GenerateHourlyData(wc, meta); if (pointpair.Count > 0) { AddChartToMaster(master, pointpair, wc, vname, meta, false); } } //save graphic // Refigure the axis ranges for the GraphPanes zgc.AxisChange(); // Layout the GraphPanes using a default Pane Layout Bitmap b = new Bitmap(2000, 666 * weatherCollections.Count); using (Graphics g = Graphics.FromImage(b)) { master.SetLayout(g, PaneLayout.SingleColumn); } master.GetImage().Save(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\ImportAnalysis\" + filename + ".jpeg", System.Drawing.Imaging.ImageFormat.Jpeg); }
public async Task monthlySummary() { List <string> collNames = MongoTools.collectionNames(db); string firstletter = ""; string vname = ""; int stationcode = 0; string source = ""; int freq = 0; int count = 0; foreach (string collection in collNames) { //all station record collections start with an s_ string[] parts = collection.Split('_'); firstletter = parts[0]; if (firstletter == "s") { stationcode = Convert.ToInt32(parts[1]); vname = parts[4]; if (vname == "PA") { continue; } source = parts[2]; freq = Convert.ToInt32(parts[5]); if (source.Contains("IDEAM")) { source = "IDEAM"; } else { source = "NOAA"; } VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); VariableMonthly vm = new VariableMonthly(vname, freq); addStation(stationcode); addMonthlyRecord(stationcode, vm); await sortByDate(stationcode, collection, meta); count++; } } insertManyMonthlySummary(); }
private async Task monthsPerVariable(string city, string variable, int month, int nCharts) { string[] months = { "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec" }; string title = city + "_" + variable + "_" + months[month - 1]; ZedGraphControl zgc = new ZedGraphControl(); MasterPane master = zgc.MasterPane; MasterPane mp = setMaster(ref master, 2000, 666 * nCharts, title); foreach (string wc in weatherCollections) { string[] parts = wc.Split('_'); int stationcode = Convert.ToInt32(parts[1]); string vname = parts[4]; string source = parts[2]; int freq = Convert.ToInt32(parts[5]); if (vname == variable && source.Contains("Clean")) { if (source.Contains("IDEAM")) { source = "IDEAM"; } else { source = "NOAA"; } VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); PointPairList pointpair = new PointPairList(); pointpair = await GenerateMonthlyData(wc, month, meta); if (pointpair.Count > 0) { AddChartToMaster(master, pointpair, wc, vname, meta, true); } } } saveGraphic(zgc, master, @"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\groupMonthlyScatterCharts\" + title + ".jpeg"); }
private async Task selectDayOfYearCDF(string vcode, SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData) { var v = synthYear.variables.Find(x => x.name == vcode); var builder = Builders <RecordMongo> .Filter; string[] pieces; VariableMeta vm; List <int> missingDays = new List <int>(); List <List <RecordMongo> > possDayValues; //find collections with current variable List <IMongoCollection <RecordMongo> > sourceStationData = new List <IMongoCollection <RecordMongo> >(); foreach (IMongoCollection <RecordMongo> sd in stationData) { pieces = sd.CollectionNamespace.CollectionName.Split('_'); if (pieces[4] == vcode) { sourceStationData.Add(sd); } } for (int doy = 1; doy < 366; doy++) { //each day will have several canadidate days sourced from each collection of same variable possDayValues = new List <List <RecordMongo> >(); foreach (IMongoCollection <RecordMongo> sd in sourceStationData) { pieces = sd.CollectionNamespace.CollectionName.Split('_'); string source = pieces[2]; if (source.Contains("NOAA")) { source = "NOAA"; } else { source = "IDEAM"; } vm = AnnualSummary.getVariableMetaFromDB(vcode, source, db); var project = BsonDocument.Parse( "{value: '$value',time:'$time',dayOfYear: {$dayOfYear: '$time'},year: {$year: '$time'}}"); try { //.Match(BsonDocument.Parse("{'dayOfYear' : {$eq : " + doy.ToString() + "}}")) var aggregationDocument = sd.Aggregate() .Unwind("value") .Project(project) .Match(BsonDocument.Parse("{$and:[" + "{'dayOfYear' : {$eq : " + doy.ToString() + "}}" + ",{'value':{$lte:" + vm.max.ToString() + " }}" + ",{'value':{$gte:" + vm.min.ToString() + "}}]}")) .ToList(); IEnumerable <IGrouping <int, BsonDocument> > query = aggregationDocument.GroupBy( doc => doc.GetValue("year").ToInt32(), doc => doc); foreach (IGrouping <int, BsonDocument> yearDayGroup in query) { var year = yearDayGroup.Key; var hours = yearDayGroup.Count(); //one group per day per year count should be 24 //but many noaa data are sometimes day time only 6-6 12 readings if (hours >= 12) { List <RecordMongo> dayValues = new List <RecordMongo>(); foreach (BsonDocument name in yearDayGroup) { RecordMongo rm = new RecordMongo(); double value = name.GetValue("value").ToDouble(); //check nub and HRs are in the right range if (vcode == "HR" && value <= 1) { value = value * 100; } if (vcode == "NUB") { //noaa's cloud is oktas if (value == 9) { value = 10; } else { value = (int)(value / 8.0 * 10); } } rm.value = value; rm.time = name.GetValue("time").ToLocalTime(); dayValues.Add(rm); } possDayValues.Add(dayValues); } } } catch (Exception e) { this.addLineToLogFile("WARN: " + synthYear.name + vcode + " error finding cdf day at day of year: " + doy); } } if (possDayValues.Count > 0) { List <RecordMongo> dayToInsert = typicalDay(possDayValues, vcode); addValuesToSynthYear(dayToInsert, ref v, doy, vcode); } else { //no possible days found empty day missingDays.Add(doy); } } if (missingDays.Count > 0) { fillMissingDays(missingDays); } //fill missing days }
private async Task generateHour2(string vcode, SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData, string meanMedian) { var v = synthYear.variables.Find(x => x.name == vcode); var builder = Builders <RecordMongo> .Filter; string[] pieces; VariableMeta vm; int hourofsyntheticyear = 0; DateTime universal = new DateTime(); //find collections with current variable List <IMongoCollection <RecordMongo> > sourceStationData = new List <IMongoCollection <RecordMongo> >(); foreach (IMongoCollection <RecordMongo> sd in stationData) { pieces = sd.CollectionNamespace.CollectionName.Split('_'); if (pieces[4] == vcode) { sourceStationData.Add(sd); } } foreach (RecordMongo r in v.records) { //synth year is local time universal = r.time.ToUniversalTime(); int h = universal.Hour; int doy = universal.DayOfYear; hourofsyntheticyear++; int foundValues = 0; List <double> valuesForHour = new List <double>(); foreach (IMongoCollection <RecordMongo> sd in sourceStationData) { pieces = sd.CollectionNamespace.CollectionName.Split('_'); string source = pieces[2]; if (source.Contains("NOAA")) { source = "NOAA"; } else { source = "IDEAM"; } vm = AnnualSummary.getVariableMetaFromDB(vcode, source, db); var project = BsonDocument.Parse( "{value: '$value',time:'$time',dayOfYear: {$dayOfYear: '$time'},hour: {$hour: '$time'}}"); try { var aggregationDocument = sd.Aggregate() .Unwind("value") .Project(project) .Match(BsonDocument.Parse("{$and:[" + "{'dayOfYear' : {$eq : " + doy.ToString() + "}}" + ",{'hour' : {$eq : " + h.ToString() + "}}" + ",{'value':{$lte:" + vm.max.ToString() + " }}" + ",{'value':{$gte:" + vm.min.ToString() + "}}]}")) .ToList(); IEnumerable <IGrouping <int, BsonDocument> > query = aggregationDocument.GroupBy( doc => doc.GetValue("dayOfYear").ToInt32(), doc => doc); foreach (IGrouping <int, BsonDocument> hourValsGroup in query) { foreach (BsonDocument name in hourValsGroup) { double value = name.GetValue("value").ToDouble(); foundValues++; //check nub and HRs are in the right range if (vcode == "HR" && value <= 1) { value = value * 100; } if (vcode == "NUB") { //noaa's cloud is oktas if (value == 9) { value = 10; } else { value = (int)(value / 8.0 * 10); } } valuesForHour.Add(value); } } } catch (Exception e) { this.addLineToLogFile("WARN: " + synthYear.name + vcode + " error finding hourly values at day of year: " + doy + " ,hour : " + h); } } if (foundValues != 0) { if (meanMedian == "meanHour") { r.value = Accord.Statistics.Measures.Mean(valuesForHour.ToArray()); } if (meanMedian == "medianHour") { r.value = Accord.Statistics.Measures.Median(valuesForHour.ToArray()); } if (meanMedian == "randomHour") { int total = valuesForHour.Count; Random rand = new Random(); r.value = valuesForHour[rand.Next(0, total)]; } } } }
private async Task generateHour(string vcode, SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData, string meanMedian) { var v = synthYear.variables.Find(x => x.name == vcode); var builder = Builders <RecordMongo> .Filter; string[] pieces; VariableMeta vm; int hourofsyntheticyear = 0; DateTime local = new DateTime(); DateTime universal = new DateTime(); //find collections with current variable List <IMongoCollection <RecordMongo> > sourceStationData = new List <IMongoCollection <RecordMongo> >(); foreach (IMongoCollection <RecordMongo> sd in stationData) { pieces = sd.CollectionNamespace.CollectionName.Split('_'); if (pieces[4] == vcode) { sourceStationData.Add(sd); } } foreach (RecordMongo r in v.records) { //this is the time we need to fill //need to filter for month day and hour int m = r.time.Month; int d = r.time.Day; int h = r.time.Hour; hourofsyntheticyear++; double value = 0; int foundValues = 0; List <double> valuesForHour = new List <double>(); foreach (IMongoCollection <RecordMongo> sd in sourceStationData) { //only if the vcode matches pieces = sd.CollectionNamespace.CollectionName.Split('_'); string source = pieces[2]; vm = AnnualSummary.getVariableMetaFromDB(vcode, source, db); int startYr = 0; int endYr = 0; getFirstLastYear(sd, ref startYr, ref endYr); if (startYr == 1) { startYr = 1980; } if (endYr == 1) { startYr = 2018; } for (int y = startYr; y < endYr; y++) { local = new DateTime(y, m, d, h, 0, 0); universal = local.ToUniversalTime(); var filter = builder.Eq("time", universal) & builder.Gte("value", vm.min) & builder.Lte("value", vm.max); //some collections have duplicate timestamps! using (IAsyncCursor <RecordMongo> cursor = await sd.FindAsync(filter)) { while (await cursor.MoveNextAsync()) { IEnumerable <RecordMongo> documents = cursor.Current; //insert into the station collection foreach (RecordMongo sdrm in documents) { value = sdrm.value; if (vcode == "HR" && sdrm.value <= 1) { value = value * 100; } if (vcode == "NUB") { if (value == 9) { value = 10; } value = (int)(value / 8.0 * 10); } valuesForHour.Add(value); foundValues++; } } } } } if (foundValues != 0) { if (meanMedian == "meanHour") { r.value = Accord.Statistics.Measures.Mean(valuesForHour.ToArray()); } if (meanMedian == "medianHour") { r.value = Accord.Statistics.Measures.Median(valuesForHour.ToArray()); } if (meanMedian == "randomHour") { int total = valuesForHour.Count; Random rand = new Random(); r.value = valuesForHour[rand.Next(0, total)]; } } } }