public async Task removeRecordsOutsideRange(int scode, string collname, VariableMeta vm, string newcollectionname) { IMongoCollection <RecordMongo> collection = db.GetCollection <RecordMongo>(collname); var filter = FilterDefinition <RecordMongo> .Empty; FindOptions <RecordMongo> options = new FindOptions <RecordMongo> { BatchSize = 1000, NoCursorTimeout = true }; using (IAsyncCursor <RecordMongo> cursor = await collection.FindAsync(filter, options)) { while (await cursor.MoveNextAsync()) { IEnumerable <RecordMongo> batch = cursor.Current; foreach (RecordMongo rm in batch) { //only if the value is in range if (rm.value > vm.min && rm.value < vm.max) { addRecord(rm, newcollectionname); } } } } }
public void convertSingleCollection(string collection) { string[] parts = collection.Split('_'); int stationcode = Convert.ToInt32(parts[1]); string vname = parts[4]; string source = parts[2]; int freq = Convert.ToInt32(parts[5]); if (freq == 60) { return; } VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); string newname = convertNameTo60min(collection); //collection for the avergaed data CollectionMongo cm = new CollectionMongo(); cm.name = newname; newAveragedData.Add(cm); Task t1 = Task.Run(() => sortByDateAndAverage(stationcode, collection, meta, newname)); t1.Wait(); insertMany(cm.records, cm.name); }
public void cleanUp() { List <string> collNames = MongoTools.collectionNames(db); string vname = ""; int stationcode = 0; string source = ""; int freq = 0; foreach (string collection in collNames) { //all station record collections start with an s_ if (collection[0] == 's') { string[] parts = collection.Split('_'); stationcode = Convert.ToInt32(parts[1]); vname = parts[4]; if (vname == "PA") { continue; } source = parts[2]; freq = Convert.ToInt32(parts[5]); VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); if (freq == 60) { string newname = convertNameToClean(collection); CollectionMongo cm = new CollectionMongo(); cm.name = newname; removeRecordsOutsideRange(stationcode, collection, meta, newname); } } } }
public async Task sortByDate(int sCode, string collname, VariableMeta vm) { IMongoCollection <RecordMongo> collection = db.GetCollection <RecordMongo>(collname); var filter = FilterDefinition <RecordMongo> .Empty; var sorter = Builders <RecordMongo> .Sort.Ascending("time"); FindOptions <RecordMongo> options = new FindOptions <RecordMongo> { BatchSize = 1000, NoCursorTimeout = true }; using (IAsyncCursor <RecordMongo> cursor = await collection.FindAsync(filter, options)) { while (await cursor.MoveNextAsync()) { IEnumerable <RecordMongo> batch = cursor.Current; foreach (RecordMongo rm in batch) { if (rm.value > vm.min && rm.value < vm.max) { //inside the range add to month total incrementMonthTotal(sCode, vm.code, rm.time.Year, rm.time.Month); } } } } }
public void cleanSingle(string collection) { string[] parts = collection.Split('_'); int stationcode = Convert.ToInt32(parts[1]); string vname = parts[4]; if (vname == "PA") { return; } string source = parts[2]; int freq = Convert.ToInt32(parts[5]); VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); string newname = convertNameToClean(collection); //collection for the avergaed data CollectionMongo cm = new CollectionMongo(); cm.name = newname; newCleanData.Add(cm); Task t1 = Task.Run(() => removeRecordsOutsideRange(stationcode, collection, meta, newname)); t1.Wait(); insertMany(cm.records, cm.name); }
private async Task <PointPairList> GenerateHourlyData(string collname, VariableMeta vm) { PointPairList list = new PointPairList(); IMongoCollection <RecordMongo> collection = db.GetCollection <RecordMongo>(collname); var filter = FilterDefinition <RecordMongo> .Empty; FindOptions <RecordMongo> options = new FindOptions <RecordMongo> { BatchSize = 1000, NoCursorTimeout = true }; using (IAsyncCursor <RecordMongo> cursor = await collection.FindAsync(filter, options)) { while (await cursor.MoveNextAsync()) { IEnumerable <RecordMongo> batch = cursor.Current; foreach (RecordMongo rm in batch) { //only if the value is in range if (rm.value > vm.min && rm.value < vm.max) { list.Add(rm.time.Hour, rm.value); } } } } return(list); }
private async Task cityGroupGraphic(StationGroup cityGroup, List <string> weatherCollections, string filename) { List <City> cities = MapTools.readCities(); var c = cities.Find(x => x.name == cityGroup.name); //start end dates //set the master pane ZedGraphControl zgc = new ZedGraphControl(); MasterPane master = zgc.MasterPane; master.Rect = new RectangleF(0, 0, 2000, 666 * weatherCollections.Count); master.PaneList.Clear(); master.Title.Text = "City: " + c.name + " lat: " + Math.Round(c.location[1], 3) + " lon: " + Math.Round(c.location[0], 3) + " alt: " + (int)(c.elevation);// + "\nDate range: " + startDate.Year + "_" + startDate.Month + " >> " + endDate.Year + "_" + endDate.Month; master.Title.FontSpec = new FontSpec("Arial", 7.0f, Color.Black, false, false, false); master.Margin.All = 5; master.Legend.IsVisible = false; int stationcode = 0; string vname = ""; string source = ""; int freq = 0; foreach (string wc in weatherCollections) { ////create one scatter for each stationvariable if (!wc.Contains("Clean")) { string[] parts = wc.Split('_'); stationcode = Convert.ToInt32(parts[1]); vname = parts[4]; source = parts[2]; freq = Convert.ToInt32(parts[5]); VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); PointPairList pointpair = new PointPairList(); pointpair = await GenerateHourlyData(wc, meta); if (pointpair.Count > 0) { AddChartToMaster(master, pointpair, wc, vname, meta, false); } } } //save graphic // Refigure the axis ranges for the GraphPanes zgc.AxisChange(); // Layout the GraphPanes using a default Pane Layout Bitmap b = new Bitmap(2000, 666 * weatherCollections.Count); using (Graphics g = Graphics.FromImage(b)) { master.SetLayout(g, PaneLayout.SingleColumn); } master.GetImage().Save(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\IDEAM\DailyAnalysis\" + filename + ".jpeg", System.Drawing.Imaging.ImageFormat.Jpeg); }
private async Task graphSingleStation(string collection, string filename) { //set the master pane ZedGraphControl zgc = new ZedGraphControl(); MasterPane master = zgc.MasterPane; master.Rect = new RectangleF(0, 0, 2000, 666); master.PaneList.Clear(); //master.Title.Text = "City: " + c.name + // " lat: " + Math.Round(c.location[1], 3) + " lon: " + Math.Round(c.location[0], 3) + // " alt: " + (int)(c.elevation);// + "\nDate range: " + startDate.Year + "_" + startDate.Month + " >> " + endDate.Year + "_" + endDate.Month; master.Title.FontSpec = new FontSpec("Arial", 7.0f, Color.Black, false, false, false); master.Margin.All = 5; master.Legend.IsVisible = false; int stationcode = 0; string vname = ""; string source = ""; int freq = 0; string[] parts = collection.Split('_'); stationcode = Convert.ToInt32(parts[1]); vname = parts[4]; source = parts[2]; freq = Convert.ToInt32(parts[5]); VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); PointPairList pointpair = new PointPairList(); pointpair = await GenerateHourlyData(collection, meta); if (pointpair.Count > 0) { AddChartToMaster(master, pointpair, collection, vname, meta, false); } //save graphic // Refigure the axis ranges for the GraphPanes zgc.AxisChange(); // Layout the GraphPanes using a default Pane Layout Bitmap b = new Bitmap(2000, 666); using (Graphics g = Graphics.FromImage(b)) { master.SetLayout(g, PaneLayout.SingleColumn); } master.GetImage().Save(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\IDEAM\DailyAnalysisTS_RS\" + filename + ".jpeg", System.Drawing.Imaging.ImageFormat.Jpeg); }
public async Task <int> insideRange(string collname, int sCode, VariableMeta meta) { IMongoCollection <BsonDocument> collection = db.GetCollection <BsonDocument>(collname); var builder = Builders <BsonDocument> .Filter; var filter = builder.Eq("stationCode", sCode) & builder.Gte("value", meta.min) & builder.Lte("value", meta.max); FindOptions <BsonDocument> options = new FindOptions <BsonDocument> { BatchSize = 1000, NoCursorTimeout = false }; var total = await collection.CountDocumentsAsync(filter); updateRecordInside(sCode, meta.code, (int)total); return(1); }
private async Task testDaysGraphic(List <string> weatherCollections, string filename) { //set the master pane ZedGraphControl zgc = new ZedGraphControl(); MasterPane master = zgc.MasterPane; master.Rect = new RectangleF(0, 0, 2000, 666 * weatherCollections.Count); master.PaneList.Clear(); master.Title.FontSpec = new FontSpec("Arial", 7.0f, Color.Black, false, false, false); master.Margin.All = 5; master.Legend.IsVisible = false; int stationcode = 0; string vname = ""; string source = ""; int freq = 0; foreach (string wc in weatherCollections) { ////create one scatter for each stationvariable string[] parts = wc.Split('_'); stationcode = Convert.ToInt32(parts[1]); vname = parts[4]; source = parts[2]; freq = Convert.ToInt32(parts[5]); VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); PointPairList pointpair = new PointPairList(); pointpair = await GenerateHourlyData(wc, meta); if (pointpair.Count > 0) { AddChartToMaster(master, pointpair, wc, vname, meta, false); } } //save graphic // Refigure the axis ranges for the GraphPanes zgc.AxisChange(); // Layout the GraphPanes using a default Pane Layout Bitmap b = new Bitmap(2000, 666 * weatherCollections.Count); using (Graphics g = Graphics.FromImage(b)) { master.SetLayout(g, PaneLayout.SingleColumn); } master.GetImage().Save(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\ImportAnalysis\" + filename + ".jpeg", System.Drawing.Imaging.ImageFormat.Jpeg); }
public async Task monthlySummary() { List <string> collNames = MongoTools.collectionNames(db); string firstletter = ""; string vname = ""; int stationcode = 0; string source = ""; int freq = 0; int count = 0; foreach (string collection in collNames) { //all station record collections start with an s_ string[] parts = collection.Split('_'); firstletter = parts[0]; if (firstletter == "s") { stationcode = Convert.ToInt32(parts[1]); vname = parts[4]; if (vname == "PA") { continue; } source = parts[2]; freq = Convert.ToInt32(parts[5]); if (source.Contains("IDEAM")) { source = "IDEAM"; } else { source = "NOAA"; } VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); VariableMonthly vm = new VariableMonthly(vname, freq); addStation(stationcode); addMonthlyRecord(stationcode, vm); await sortByDate(stationcode, collection, meta); count++; } } insertManyMonthlySummary(); }
public void getRequiredVaribleMeta() { StreamReader sr = new StreamReader(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\VariablesMeta.csv"); string line = sr.ReadLine(); string[] parts; VariableMeta meta; while (line != null) { parts = line.Split(','); meta = new VariableMeta(parts[0], parts[1], parts[3], Convert.ToInt32(parts[6]), Convert.ToInt32(parts[5]), parts[2], Convert.ToInt32(parts[4])); variableMeta.Add(meta); line = sr.ReadLine(); } sr.Close(); insertManyRecord(); }
public async Task textSummaryStations(string dbname) { //summary based on total variables List <string> collNames = MongoTools.collectionNames(db); string firstletter = ""; string vname = ""; int stationcode = 0; string source = ""; int freq = 0; foreach (string collection in collNames) { //all station record collections start with an s_ string[] parts = collection.Split('_'); firstletter = parts[0]; if (firstletter == "s") { stationcode = Convert.ToInt32(parts[1]); vname = parts[4]; if (vname == "PA") { continue; } source = parts[2]; freq = Convert.ToInt32(parts[5]); VariableMeta meta = getVariableMetaFromDB(vname, source, db); RecordMeta rm = new RecordMeta(vname, freq); addStation(stationcode); addRecord(stationcode, rm); await getTotalRecords(collection, stationcode, vname); await getDateLimits(collection, stationcode, vname); await insideRange(collection, stationcode, meta); } } annualStats(); insertManyRecordStationSummary(); printToSummary(); }
private async Task monthsPerVariable(string city, string variable, int month, int nCharts) { string[] months = { "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec" }; string title = city + "_" + variable + "_" + months[month - 1]; ZedGraphControl zgc = new ZedGraphControl(); MasterPane master = zgc.MasterPane; MasterPane mp = setMaster(ref master, 2000, 666 * nCharts, title); foreach (string wc in weatherCollections) { string[] parts = wc.Split('_'); int stationcode = Convert.ToInt32(parts[1]); string vname = parts[4]; string source = parts[2]; int freq = Convert.ToInt32(parts[5]); if (vname == variable && source.Contains("Clean")) { if (source.Contains("IDEAM")) { source = "IDEAM"; } else { source = "NOAA"; } VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db); PointPairList pointpair = new PointPairList(); pointpair = await GenerateMonthlyData(wc, month, meta); if (pointpair.Count > 0) { AddChartToMaster(master, pointpair, wc, vname, meta, true); } } } saveGraphic(zgc, master, @"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\groupMonthlyScatterCharts\" + title + ".jpeg"); }
private void AddChartToMaster(MasterPane master, PointPairList list, string title, string yaxistitle, VariableMeta meta, bool addStats) { GraphPane pane = new GraphPane(); // Set the titles pane.Title.Text = title; pane.XAxis.Title.Text = "hour"; pane.YAxis.Title.Text = yaxistitle; pane.Border.IsVisible = false; pane.XAxis.Scale.Max = 23; pane.XAxis.Scale.Min = 0; pane.XAxis.Scale.MajorStep = 4.0; pane.XAxis.Scale.MinorStep = 1.0; pane.YAxis.Scale.Min = meta.min; pane.YAxis.Scale.Max = meta.max; // Add the curve LineItem myCurve = pane.AddCurve(yaxistitle, list, Color.Black, SymbolType.XCross); // Don't display the line (This makes a scatter plot) myCurve.Line.IsVisible = false; // Hide the symbol outline myCurve.Symbol.Border.IsVisible = true; // Fill the symbol interior with color myCurve.Symbol.Size = 2f; if (addStats) { addStatsToPane(list, pane); } pane.AxisChange(); master.Add(pane); }
private async Task <PointPairList> GenerateMonthlyData(string collname, int month, VariableMeta vm) { //testAggreate(collname); PointPairList list = new PointPairList(); IMongoCollection <RecordMongo> collection = db.GetCollection <RecordMongo>(collname); var project = BsonDocument.Parse( "{value: '$value',time:'$time',month: {$month: '$time'}}"); try { var aggregationDocument = collection.Aggregate() .Unwind("value") .Project(project) .Match(BsonDocument.Parse("{$and:[{'month' : {$eq : " + month.ToString() + "}},{'value':{$lte:" + vm.max.ToString() + " }},{'value':{$gte:" + vm.min.ToString() + "}}]}")) .ToList(); if (aggregationDocument != null) { foreach (var result in aggregationDocument) { //Console.WriteLine(result.ToString()); var hour = result.GetValue("time").ToLocalTime().Hour; list.Add(hour, result.GetValue("value").ToDouble()); } } } catch (Exception e) { var error = "errorhere"; } return(list); }
public async Task sortByDateAndAverage(int scode, string collname, VariableMeta vm, string newcollectionname) { IMongoCollection <RecordMongo> collection = db.GetCollection <RecordMongo>(collname); var filter = FilterDefinition <RecordMongo> .Empty; var sorter = Builders <RecordMongo> .Sort.Ascending("time"); FindOptions <RecordMongo> options = new FindOptions <RecordMongo> { BatchSize = 500, NoCursorTimeout = false, Sort = sorter }; DateTime currentHour = new DateTime(); bool firstrecord = true; using (IAsyncCursor <RecordMongo> cursor = await collection.FindAsync(filter, options)) { double hourtotal = 0; int recordsPerHr = 0; while (await cursor.MoveNextAsync()) { IEnumerable <RecordMongo> batch = cursor.Current; foreach (RecordMongo rm in batch) { //only if the value is in range if (rm.value > vm.min && rm.value < vm.max) { if (firstrecord) { currentHour = rm.time; firstrecord = false; } if (rm.time.DayOfYear == currentHour.DayOfYear && rm.time.Hour == currentHour.Hour) { recordsPerHr++; hourtotal += rm.value; } else { //make a new record and add to the list RecordMongo avrm = new RecordMongo(); avrm.processNote = "averaged from 10min readings"; avrm.stationCode = scode; avrm.time = new DateTime(currentHour.Year, currentHour.Month, currentHour.Day, currentHour.Hour, 0, 0); avrm.value = hourtotal / recordsPerHr; if (!Double.IsNaN(avrm.value)) { addRecord(avrm, newcollectionname); } //reset the counter and total recordsPerHr = 0; hourtotal = 0; //set the new hour currentHour = rm.time; } } } } } }