public async Task removeRecordsOutsideRange(int scode, string collname, VariableMeta vm, string newcollectionname)
{
IMongoCollection <RecordMongo> collection = db.GetCollection <RecordMongo>(collname);
var filter = FilterDefinition <RecordMongo> .Empty;
FindOptions <RecordMongo> options = new FindOptions <RecordMongo>
{
BatchSize = 1000,
NoCursorTimeout = true
};
using (IAsyncCursor <RecordMongo> cursor = await collection.FindAsync(filter, options))
{
while (await cursor.MoveNextAsync())
{
IEnumerable <RecordMongo> batch = cursor.Current;
foreach (RecordMongo rm in batch)
{
//only if the value is in range
if (rm.value > vm.min && rm.value < vm.max)
{
addRecord(rm, newcollectionname);
}
}
}
}
}
public void convertSingleCollection(string collection)
{
string[] parts = collection.Split('_');
int stationcode = Convert.ToInt32(parts[1]);
string vname = parts[4];
string source = parts[2];
int freq = Convert.ToInt32(parts[5]);
if (freq == 60)
{
return;
}
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
string newname = convertNameTo60min(collection);
//collection for the avergaed data
CollectionMongo cm = new CollectionMongo();
cm.name = newname;
newAveragedData.Add(cm);
Task t1 = Task.Run(() => sortByDateAndAverage(stationcode, collection, meta, newname));
t1.Wait();
insertMany(cm.records, cm.name);
}
public void cleanUp()
{
List <string> collNames = MongoTools.collectionNames(db);
string vname = "";
int stationcode = 0;
string source = "";
int freq = 0;
foreach (string collection in collNames)
{
//all station record collections start with an s_
if (collection[0] == 's')
{
string[] parts = collection.Split('_');
stationcode = Convert.ToInt32(parts[1]);
vname = parts[4];
if (vname == "PA")
{
continue;
}
source = parts[2];
freq = Convert.ToInt32(parts[5]);
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
if (freq == 60)
{
string newname = convertNameToClean(collection);
CollectionMongo cm = new CollectionMongo();
cm.name = newname;
removeRecordsOutsideRange(stationcode, collection, meta, newname);
}
}
}
}
public async Task sortByDate(int sCode, string collname, VariableMeta vm)
{
IMongoCollection <RecordMongo> collection = db.GetCollection <RecordMongo>(collname);
var filter = FilterDefinition <RecordMongo> .Empty;
var sorter = Builders <RecordMongo> .Sort.Ascending("time");
FindOptions <RecordMongo> options = new FindOptions <RecordMongo>
{
BatchSize = 1000,
NoCursorTimeout = true
};
using (IAsyncCursor <RecordMongo> cursor = await collection.FindAsync(filter, options))
{
while (await cursor.MoveNextAsync())
{
IEnumerable <RecordMongo> batch = cursor.Current;
foreach (RecordMongo rm in batch)
{
if (rm.value > vm.min && rm.value < vm.max)
{
//inside the range add to month total
incrementMonthTotal(sCode, vm.code, rm.time.Year, rm.time.Month);
}
}
}
}
}
public void cleanSingle(string collection)
{
string[] parts = collection.Split('_');
int stationcode = Convert.ToInt32(parts[1]);
string vname = parts[4];
if (vname == "PA")
{
return;
}
string source = parts[2];
int freq = Convert.ToInt32(parts[5]);
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
string newname = convertNameToClean(collection);
//collection for the avergaed data
CollectionMongo cm = new CollectionMongo();
cm.name = newname;
newCleanData.Add(cm);
Task t1 = Task.Run(() => removeRecordsOutsideRange(stationcode, collection, meta, newname));
t1.Wait();
insertMany(cm.records, cm.name);
}
private async Task <PointPairList> GenerateHourlyData(string collname, VariableMeta vm)
{
PointPairList list = new PointPairList();
IMongoCollection <RecordMongo> collection = db.GetCollection <RecordMongo>(collname);
var filter = FilterDefinition <RecordMongo> .Empty;
FindOptions <RecordMongo> options = new FindOptions <RecordMongo>
{
BatchSize = 1000,
NoCursorTimeout = true
};
using (IAsyncCursor <RecordMongo> cursor = await collection.FindAsync(filter, options))
{
while (await cursor.MoveNextAsync())
{
IEnumerable <RecordMongo> batch = cursor.Current;
foreach (RecordMongo rm in batch)
{
//only if the value is in range
if (rm.value > vm.min && rm.value < vm.max)
{
list.Add(rm.time.Hour, rm.value);
}
}
}
}
return(list);
}
private async Task cityGroupGraphic(StationGroup cityGroup, List <string> weatherCollections, string filename)
{
List <City> cities = MapTools.readCities();
var c = cities.Find(x => x.name == cityGroup.name);
//start end dates
//set the master pane
ZedGraphControl zgc = new ZedGraphControl();
MasterPane master = zgc.MasterPane;
master.Rect = new RectangleF(0, 0, 2000, 666 * weatherCollections.Count);
master.PaneList.Clear();
master.Title.Text = "City: " + c.name +
" lat: " + Math.Round(c.location[1], 3) + " lon: " + Math.Round(c.location[0], 3) +
" alt: " + (int)(c.elevation);// + "\nDate range: " + startDate.Year + "_" + startDate.Month + " >> " + endDate.Year + "_" + endDate.Month;
master.Title.FontSpec = new FontSpec("Arial", 7.0f, Color.Black, false, false, false);
master.Margin.All = 5;
master.Legend.IsVisible = false;
int stationcode = 0;
string vname = "";
string source = "";
int freq = 0;
foreach (string wc in weatherCollections)
{
////create one scatter for each stationvariable
if (!wc.Contains("Clean"))
{
string[] parts = wc.Split('_');
stationcode = Convert.ToInt32(parts[1]);
vname = parts[4];
source = parts[2];
freq = Convert.ToInt32(parts[5]);
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
PointPairList pointpair = new PointPairList();
pointpair = await GenerateHourlyData(wc, meta);
if (pointpair.Count > 0)
{
AddChartToMaster(master, pointpair, wc, vname, meta, false);
}
}
}
//save graphic
// Refigure the axis ranges for the GraphPanes
zgc.AxisChange();
// Layout the GraphPanes using a default Pane Layout
Bitmap b = new Bitmap(2000, 666 * weatherCollections.Count);
using (Graphics g = Graphics.FromImage(b))
{
master.SetLayout(g, PaneLayout.SingleColumn);
}
master.GetImage().Save(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\IDEAM\DailyAnalysis\" + filename + ".jpeg", System.Drawing.Imaging.ImageFormat.Jpeg);
}
private async Task graphSingleStation(string collection, string filename)
{
//set the master pane
ZedGraphControl zgc = new ZedGraphControl();
MasterPane master = zgc.MasterPane;
master.Rect = new RectangleF(0, 0, 2000, 666);
master.PaneList.Clear();
//master.Title.Text = "City: " + c.name +
// " lat: " + Math.Round(c.location[1], 3) + " lon: " + Math.Round(c.location[0], 3) +
// " alt: " + (int)(c.elevation);// + "\nDate range: " + startDate.Year + "_" + startDate.Month + " >> " + endDate.Year + "_" + endDate.Month;
master.Title.FontSpec = new FontSpec("Arial", 7.0f, Color.Black, false, false, false);
master.Margin.All = 5;
master.Legend.IsVisible = false;
int stationcode = 0;
string vname = "";
string source = "";
int freq = 0;
string[] parts = collection.Split('_');
stationcode = Convert.ToInt32(parts[1]);
vname = parts[4];
source = parts[2];
freq = Convert.ToInt32(parts[5]);
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
PointPairList pointpair = new PointPairList();
pointpair = await GenerateHourlyData(collection, meta);
if (pointpair.Count > 0)
{
AddChartToMaster(master, pointpair, collection, vname, meta, false);
}
//save graphic
// Refigure the axis ranges for the GraphPanes
zgc.AxisChange();
// Layout the GraphPanes using a default Pane Layout
Bitmap b = new Bitmap(2000, 666);
using (Graphics g = Graphics.FromImage(b))
{
master.SetLayout(g, PaneLayout.SingleColumn);
}
master.GetImage().Save(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\IDEAM\DailyAnalysisTS_RS\" + filename + ".jpeg", System.Drawing.Imaging.ImageFormat.Jpeg);
}
public async Task <int> insideRange(string collname, int sCode, VariableMeta meta)
{
IMongoCollection <BsonDocument> collection = db.GetCollection <BsonDocument>(collname);
var builder = Builders <BsonDocument> .Filter;
var filter = builder.Eq("stationCode", sCode) & builder.Gte("value", meta.min) & builder.Lte("value", meta.max);
FindOptions <BsonDocument> options = new FindOptions <BsonDocument>
{
BatchSize = 1000,
NoCursorTimeout = false
};
var total = await collection.CountDocumentsAsync(filter);
updateRecordInside(sCode, meta.code, (int)total);
return(1);
}
private async Task testDaysGraphic(List <string> weatherCollections, string filename)
{
//set the master pane
ZedGraphControl zgc = new ZedGraphControl();
MasterPane master = zgc.MasterPane;
master.Rect = new RectangleF(0, 0, 2000, 666 * weatherCollections.Count);
master.PaneList.Clear();
master.Title.FontSpec = new FontSpec("Arial", 7.0f, Color.Black, false, false, false);
master.Margin.All = 5;
master.Legend.IsVisible = false;
int stationcode = 0;
string vname = "";
string source = "";
int freq = 0;
foreach (string wc in weatherCollections)
{
////create one scatter for each stationvariable
string[] parts = wc.Split('_');
stationcode = Convert.ToInt32(parts[1]);
vname = parts[4];
source = parts[2];
freq = Convert.ToInt32(parts[5]);
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
PointPairList pointpair = new PointPairList();
pointpair = await GenerateHourlyData(wc, meta);
if (pointpair.Count > 0)
{
AddChartToMaster(master, pointpair, wc, vname, meta, false);
}
}
//save graphic
// Refigure the axis ranges for the GraphPanes
zgc.AxisChange();
// Layout the GraphPanes using a default Pane Layout
Bitmap b = new Bitmap(2000, 666 * weatherCollections.Count);
using (Graphics g = Graphics.FromImage(b))
{
master.SetLayout(g, PaneLayout.SingleColumn);
}
master.GetImage().Save(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\ImportAnalysis\" + filename + ".jpeg", System.Drawing.Imaging.ImageFormat.Jpeg);
}
public async Task monthlySummary()
{
List <string> collNames = MongoTools.collectionNames(db);
string firstletter = "";
string vname = "";
int stationcode = 0;
string source = "";
int freq = 0;
int count = 0;
foreach (string collection in collNames)
{
//all station record collections start with an s_
string[] parts = collection.Split('_');
firstletter = parts[0];
if (firstletter == "s")
{
stationcode = Convert.ToInt32(parts[1]);
vname = parts[4];
if (vname == "PA")
{
continue;
}
source = parts[2];
freq = Convert.ToInt32(parts[5]);
if (source.Contains("IDEAM"))
{
source = "IDEAM";
}
else
{
source = "NOAA";
}
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
VariableMonthly vm = new VariableMonthly(vname, freq);
addStation(stationcode);
addMonthlyRecord(stationcode, vm);
await sortByDate(stationcode, collection, meta);
count++;
}
}
insertManyMonthlySummary();
}
public void getRequiredVaribleMeta()
{
StreamReader sr = new StreamReader(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\VariablesMeta.csv");
string line = sr.ReadLine();
string[] parts;
VariableMeta meta;
while (line != null)
{
parts = line.Split(',');
meta = new VariableMeta(parts[0], parts[1], parts[3], Convert.ToInt32(parts[6]), Convert.ToInt32(parts[5]), parts[2], Convert.ToInt32(parts[4]));
variableMeta.Add(meta);
line = sr.ReadLine();
}
sr.Close();
insertManyRecord();
}
public async Task textSummaryStations(string dbname)
{
//summary based on total variables
List <string> collNames = MongoTools.collectionNames(db);
string firstletter = "";
string vname = "";
int stationcode = 0;
string source = "";
int freq = 0;
foreach (string collection in collNames)
{
//all station record collections start with an s_
string[] parts = collection.Split('_');
firstletter = parts[0];
if (firstletter == "s")
{
stationcode = Convert.ToInt32(parts[1]);
vname = parts[4];
if (vname == "PA")
{
continue;
}
source = parts[2];
freq = Convert.ToInt32(parts[5]);
VariableMeta meta = getVariableMetaFromDB(vname, source, db);
RecordMeta rm = new RecordMeta(vname, freq);
addStation(stationcode);
addRecord(stationcode, rm);
await getTotalRecords(collection, stationcode, vname);
await getDateLimits(collection, stationcode, vname);
await insideRange(collection, stationcode, meta);
}
}
annualStats();
insertManyRecordStationSummary();
printToSummary();
}
private async Task monthsPerVariable(string city, string variable, int month, int nCharts)
{
string[] months = { "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec" };
string title = city + "_" + variable + "_" + months[month - 1];
ZedGraphControl zgc = new ZedGraphControl();
MasterPane master = zgc.MasterPane;
MasterPane mp = setMaster(ref master, 2000, 666 * nCharts, title);
foreach (string wc in weatherCollections)
{
string[] parts = wc.Split('_');
int stationcode = Convert.ToInt32(parts[1]);
string vname = parts[4];
string source = parts[2];
int freq = Convert.ToInt32(parts[5]);
if (vname == variable && source.Contains("Clean"))
{
if (source.Contains("IDEAM"))
{
source = "IDEAM";
}
else
{
source = "NOAA";
}
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
PointPairList pointpair = new PointPairList();
pointpair = await GenerateMonthlyData(wc, month, meta);
if (pointpair.Count > 0)
{
AddChartToMaster(master, pointpair, wc, vname, meta, true);
}
}
}
saveGraphic(zgc, master, @"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\groupMonthlyScatterCharts\" + title + ".jpeg");
}
private void AddChartToMaster(MasterPane master, PointPairList list, string title, string yaxistitle, VariableMeta meta, bool addStats)
{
GraphPane pane = new GraphPane();
// Set the titles
pane.Title.Text = title;
pane.XAxis.Title.Text = "hour";
pane.YAxis.Title.Text = yaxistitle;
pane.Border.IsVisible = false;
pane.XAxis.Scale.Max = 23;
pane.XAxis.Scale.Min = 0;
pane.XAxis.Scale.MajorStep = 4.0;
pane.XAxis.Scale.MinorStep = 1.0;
pane.YAxis.Scale.Min = meta.min;
pane.YAxis.Scale.Max = meta.max;
// Add the curve
LineItem myCurve = pane.AddCurve(yaxistitle, list, Color.Black, SymbolType.XCross);
// Don't display the line (This makes a scatter plot)
myCurve.Line.IsVisible = false;
// Hide the symbol outline
myCurve.Symbol.Border.IsVisible = true;
// Fill the symbol interior with color
myCurve.Symbol.Size = 2f;
if (addStats)
{
addStatsToPane(list, pane);
}
pane.AxisChange();
master.Add(pane);
}
private async Task <PointPairList> GenerateMonthlyData(string collname, int month, VariableMeta vm)
{
//testAggreate(collname);
PointPairList list = new PointPairList();
IMongoCollection <RecordMongo> collection = db.GetCollection <RecordMongo>(collname);
var project =
BsonDocument.Parse(
"{value: '$value',time:'$time',month: {$month: '$time'}}");
try {
var aggregationDocument =
collection.Aggregate()
.Unwind("value")
.Project(project)
.Match(BsonDocument.Parse("{$and:[{'month' : {$eq : " + month.ToString() + "}},{'value':{$lte:" + vm.max.ToString() + " }},{'value':{$gte:" + vm.min.ToString() + "}}]}"))
.ToList();
if (aggregationDocument != null)
{
foreach (var result in aggregationDocument)
{
//Console.WriteLine(result.ToString());
var hour = result.GetValue("time").ToLocalTime().Hour;
list.Add(hour, result.GetValue("value").ToDouble());
}
}
}
catch (Exception e)
{
var error = "errorhere";
}
return(list);
}
public async Task sortByDateAndAverage(int scode, string collname, VariableMeta vm, string newcollectionname)
{
IMongoCollection <RecordMongo> collection = db.GetCollection <RecordMongo>(collname);
var filter = FilterDefinition <RecordMongo> .Empty;
var sorter = Builders <RecordMongo> .Sort.Ascending("time");
FindOptions <RecordMongo> options = new FindOptions <RecordMongo>
{
BatchSize = 500,
NoCursorTimeout = false,
Sort = sorter
};
DateTime currentHour = new DateTime();
bool firstrecord = true;
using (IAsyncCursor <RecordMongo> cursor = await collection.FindAsync(filter, options))
{
double hourtotal = 0;
int recordsPerHr = 0;
while (await cursor.MoveNextAsync())
{
IEnumerable <RecordMongo> batch = cursor.Current;
foreach (RecordMongo rm in batch)
{
//only if the value is in range
if (rm.value > vm.min && rm.value < vm.max)
{
if (firstrecord)
{
currentHour = rm.time;
firstrecord = false;
}
if (rm.time.DayOfYear == currentHour.DayOfYear && rm.time.Hour == currentHour.Hour)
{
recordsPerHr++;
hourtotal += rm.value;
}
else
{
//make a new record and add to the list
RecordMongo avrm = new RecordMongo();
avrm.processNote = "averaged from 10min readings";
avrm.stationCode = scode;
avrm.time = new DateTime(currentHour.Year, currentHour.Month, currentHour.Day, currentHour.Hour, 0, 0);
avrm.value = hourtotal / recordsPerHr;
if (!Double.IsNaN(avrm.value))
{
addRecord(avrm, newcollectionname);
}
//reset the counter and total
recordsPerHr = 0;
hourtotal = 0;
//set the new hour
currentHour = rm.time;
}
}
}
}
}
}
