public void cleanSingle(string collection)
{
string[] parts = collection.Split('_');
int stationcode = Convert.ToInt32(parts[1]);
string vname = parts[4];
if (vname == "PA")
{
return;
}
string source = parts[2];
int freq = Convert.ToInt32(parts[5]);
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
string newname = convertNameToClean(collection);
//collection for the avergaed data
CollectionMongo cm = new CollectionMongo();
cm.name = newname;
newCleanData.Add(cm);
Task t1 = Task.Run(() => removeRecordsOutsideRange(stationcode, collection, meta, newname));
t1.Wait();
insertMany(cm.records, cm.name);
}
public void cleanUp()
{
List <string> collNames = MongoTools.collectionNames(db);
string vname = "";
int stationcode = 0;
string source = "";
int freq = 0;
foreach (string collection in collNames)
{
//all station record collections start with an s_
if (collection[0] == 's')
{
string[] parts = collection.Split('_');
stationcode = Convert.ToInt32(parts[1]);
vname = parts[4];
if (vname == "PA")
{
continue;
}
source = parts[2];
freq = Convert.ToInt32(parts[5]);
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
if (freq == 60)
{
string newname = convertNameToClean(collection);
CollectionMongo cm = new CollectionMongo();
cm.name = newname;
removeRecordsOutsideRange(stationcode, collection, meta, newname);
}
}
}
}
public void convertSingleCollection(string collection)
{
string[] parts = collection.Split('_');
int stationcode = Convert.ToInt32(parts[1]);
string vname = parts[4];
string source = parts[2];
int freq = Convert.ToInt32(parts[5]);
if (freq == 60)
{
return;
}
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
string newname = convertNameTo60min(collection);
//collection for the avergaed data
CollectionMongo cm = new CollectionMongo();
cm.name = newname;
newAveragedData.Add(cm);
Task t1 = Task.Run(() => sortByDateAndAverage(stationcode, collection, meta, newname));
t1.Wait();
insertMany(cm.records, cm.name);
}
private async Task cityGroupGraphic(StationGroup cityGroup, List <string> weatherCollections, string filename)
{
List <City> cities = MapTools.readCities();
var c = cities.Find(x => x.name == cityGroup.name);
//start end dates
//set the master pane
ZedGraphControl zgc = new ZedGraphControl();
MasterPane master = zgc.MasterPane;
master.Rect = new RectangleF(0, 0, 2000, 666 * weatherCollections.Count);
master.PaneList.Clear();
master.Title.Text = "City: " + c.name +
" lat: " + Math.Round(c.location[1], 3) + " lon: " + Math.Round(c.location[0], 3) +
" alt: " + (int)(c.elevation);// + "\nDate range: " + startDate.Year + "_" + startDate.Month + " >> " + endDate.Year + "_" + endDate.Month;
master.Title.FontSpec = new FontSpec("Arial", 7.0f, Color.Black, false, false, false);
master.Margin.All = 5;
master.Legend.IsVisible = false;
int stationcode = 0;
string vname = "";
string source = "";
int freq = 0;
foreach (string wc in weatherCollections)
{
////create one scatter for each stationvariable
if (!wc.Contains("Clean"))
{
string[] parts = wc.Split('_');
stationcode = Convert.ToInt32(parts[1]);
vname = parts[4];
source = parts[2];
freq = Convert.ToInt32(parts[5]);
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
PointPairList pointpair = new PointPairList();
pointpair = await GenerateHourlyData(wc, meta);
if (pointpair.Count > 0)
{
AddChartToMaster(master, pointpair, wc, vname, meta, false);
}
}
}
//save graphic
// Refigure the axis ranges for the GraphPanes
zgc.AxisChange();
// Layout the GraphPanes using a default Pane Layout
Bitmap b = new Bitmap(2000, 666 * weatherCollections.Count);
using (Graphics g = Graphics.FromImage(b))
{
master.SetLayout(g, PaneLayout.SingleColumn);
}
master.GetImage().Save(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\IDEAM\DailyAnalysis\" + filename + ".jpeg", System.Drawing.Imaging.ImageFormat.Jpeg);
}
private async Task graphSingleStation(string collection, string filename)
{
//set the master pane
ZedGraphControl zgc = new ZedGraphControl();
MasterPane master = zgc.MasterPane;
master.Rect = new RectangleF(0, 0, 2000, 666);
master.PaneList.Clear();
//master.Title.Text = "City: " + c.name +
// " lat: " + Math.Round(c.location[1], 3) + " lon: " + Math.Round(c.location[0], 3) +
// " alt: " + (int)(c.elevation);// + "\nDate range: " + startDate.Year + "_" + startDate.Month + " >> " + endDate.Year + "_" + endDate.Month;
master.Title.FontSpec = new FontSpec("Arial", 7.0f, Color.Black, false, false, false);
master.Margin.All = 5;
master.Legend.IsVisible = false;
int stationcode = 0;
string vname = "";
string source = "";
int freq = 0;
string[] parts = collection.Split('_');
stationcode = Convert.ToInt32(parts[1]);
vname = parts[4];
source = parts[2];
freq = Convert.ToInt32(parts[5]);
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
PointPairList pointpair = new PointPairList();
pointpair = await GenerateHourlyData(collection, meta);
if (pointpair.Count > 0)
{
AddChartToMaster(master, pointpair, collection, vname, meta, false);
}
//save graphic
// Refigure the axis ranges for the GraphPanes
zgc.AxisChange();
// Layout the GraphPanes using a default Pane Layout
Bitmap b = new Bitmap(2000, 666);
using (Graphics g = Graphics.FromImage(b))
{
master.SetLayout(g, PaneLayout.SingleColumn);
}
master.GetImage().Save(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\IDEAM\DailyAnalysisTS_RS\" + filename + ".jpeg", System.Drawing.Imaging.ImageFormat.Jpeg);
}
private async Task testDaysGraphic(List <string> weatherCollections, string filename)
{
//set the master pane
ZedGraphControl zgc = new ZedGraphControl();
MasterPane master = zgc.MasterPane;
master.Rect = new RectangleF(0, 0, 2000, 666 * weatherCollections.Count);
master.PaneList.Clear();
master.Title.FontSpec = new FontSpec("Arial", 7.0f, Color.Black, false, false, false);
master.Margin.All = 5;
master.Legend.IsVisible = false;
int stationcode = 0;
string vname = "";
string source = "";
int freq = 0;
foreach (string wc in weatherCollections)
{
////create one scatter for each stationvariable
string[] parts = wc.Split('_');
stationcode = Convert.ToInt32(parts[1]);
vname = parts[4];
source = parts[2];
freq = Convert.ToInt32(parts[5]);
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
PointPairList pointpair = new PointPairList();
pointpair = await GenerateHourlyData(wc, meta);
if (pointpair.Count > 0)
{
AddChartToMaster(master, pointpair, wc, vname, meta, false);
}
}
//save graphic
// Refigure the axis ranges for the GraphPanes
zgc.AxisChange();
// Layout the GraphPanes using a default Pane Layout
Bitmap b = new Bitmap(2000, 666 * weatherCollections.Count);
using (Graphics g = Graphics.FromImage(b))
{
master.SetLayout(g, PaneLayout.SingleColumn);
}
master.GetImage().Save(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\ImportAnalysis\" + filename + ".jpeg", System.Drawing.Imaging.ImageFormat.Jpeg);
}
public async Task monthlySummary()
{
List <string> collNames = MongoTools.collectionNames(db);
string firstletter = "";
string vname = "";
int stationcode = 0;
string source = "";
int freq = 0;
int count = 0;
foreach (string collection in collNames)
{
//all station record collections start with an s_
string[] parts = collection.Split('_');
firstletter = parts[0];
if (firstletter == "s")
{
stationcode = Convert.ToInt32(parts[1]);
vname = parts[4];
if (vname == "PA")
{
continue;
}
source = parts[2];
freq = Convert.ToInt32(parts[5]);
if (source.Contains("IDEAM"))
{
source = "IDEAM";
}
else
{
source = "NOAA";
}
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
VariableMonthly vm = new VariableMonthly(vname, freq);
addStation(stationcode);
addMonthlyRecord(stationcode, vm);
await sortByDate(stationcode, collection, meta);
count++;
}
}
insertManyMonthlySummary();
}
private async Task monthsPerVariable(string city, string variable, int month, int nCharts)
{
string[] months = { "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec" };
string title = city + "_" + variable + "_" + months[month - 1];
ZedGraphControl zgc = new ZedGraphControl();
MasterPane master = zgc.MasterPane;
MasterPane mp = setMaster(ref master, 2000, 666 * nCharts, title);
foreach (string wc in weatherCollections)
{
string[] parts = wc.Split('_');
int stationcode = Convert.ToInt32(parts[1]);
string vname = parts[4];
string source = parts[2];
int freq = Convert.ToInt32(parts[5]);
if (vname == variable && source.Contains("Clean"))
{
if (source.Contains("IDEAM"))
{
source = "IDEAM";
}
else
{
source = "NOAA";
}
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
PointPairList pointpair = new PointPairList();
pointpair = await GenerateMonthlyData(wc, month, meta);
if (pointpair.Count > 0)
{
AddChartToMaster(master, pointpair, wc, vname, meta, true);
}
}
}
saveGraphic(zgc, master, @"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\groupMonthlyScatterCharts\" + title + ".jpeg");
}
private async Task selectDayOfYearCDF(string vcode, SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData)
{
var v = synthYear.variables.Find(x => x.name == vcode);
var builder = Builders <RecordMongo> .Filter;
string[] pieces;
VariableMeta vm;
List <int> missingDays = new List <int>();
List <List <RecordMongo> > possDayValues;
//find collections with current variable
List <IMongoCollection <RecordMongo> > sourceStationData = new List <IMongoCollection <RecordMongo> >();
foreach (IMongoCollection <RecordMongo> sd in stationData)
{
pieces = sd.CollectionNamespace.CollectionName.Split('_');
if (pieces[4] == vcode)
{
sourceStationData.Add(sd);
}
}
for (int doy = 1; doy < 366; doy++)
{
//each day will have several canadidate days sourced from each collection of same variable
possDayValues = new List <List <RecordMongo> >();
foreach (IMongoCollection <RecordMongo> sd in sourceStationData)
{
pieces = sd.CollectionNamespace.CollectionName.Split('_');
string source = pieces[2];
if (source.Contains("NOAA"))
{
source = "NOAA";
}
else
{
source = "IDEAM";
}
vm = AnnualSummary.getVariableMetaFromDB(vcode, source, db);
var project =
BsonDocument.Parse(
"{value: '$value',time:'$time',dayOfYear: {$dayOfYear: '$time'},year: {$year: '$time'}}");
try
{
//.Match(BsonDocument.Parse("{'dayOfYear' : {$eq : " + doy.ToString() + "}}"))
var aggregationDocument =
sd.Aggregate()
.Unwind("value")
.Project(project)
.Match(BsonDocument.Parse("{$and:[" +
"{'dayOfYear' : {$eq : " + doy.ToString() + "}}" +
",{'value':{$lte:" + vm.max.ToString() + " }}" +
",{'value':{$gte:" + vm.min.ToString() + "}}]}"))
.ToList();
IEnumerable <IGrouping <int, BsonDocument> > query = aggregationDocument.GroupBy(
doc => doc.GetValue("year").ToInt32(),
doc => doc);
foreach (IGrouping <int, BsonDocument> yearDayGroup in query)
{
var year = yearDayGroup.Key;
var hours = yearDayGroup.Count();
//one group per day per year count should be 24
//but many noaa data are sometimes day time only 6-6 12 readings
if (hours >= 12)
{
List <RecordMongo> dayValues = new List <RecordMongo>();
foreach (BsonDocument name in yearDayGroup)
{
RecordMongo rm = new RecordMongo();
double value = name.GetValue("value").ToDouble();
//check nub and HRs are in the right range
if (vcode == "HR" && value <= 1)
{
value = value * 100;
}
if (vcode == "NUB")
{
//noaa's cloud is oktas
if (value == 9)
{
value = 10;
}
else
{
value = (int)(value / 8.0 * 10);
}
}
rm.value = value;
rm.time = name.GetValue("time").ToLocalTime();
dayValues.Add(rm);
}
possDayValues.Add(dayValues);
}
}
}
catch (Exception e)
{
this.addLineToLogFile("WARN: " + synthYear.name + vcode + " error finding cdf day at day of year: " + doy);
}
}
if (possDayValues.Count > 0)
{
List <RecordMongo> dayToInsert = typicalDay(possDayValues, vcode);
addValuesToSynthYear(dayToInsert, ref v, doy, vcode);
}
else
{
//no possible days found empty day
missingDays.Add(doy);
}
}
if (missingDays.Count > 0)
{
fillMissingDays(missingDays);
}
//fill missing days
}
private async Task generateHour2(string vcode, SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData, string meanMedian)
{
var v = synthYear.variables.Find(x => x.name == vcode);
var builder = Builders <RecordMongo> .Filter;
string[] pieces;
VariableMeta vm;
int hourofsyntheticyear = 0;
DateTime universal = new DateTime();
//find collections with current variable
List <IMongoCollection <RecordMongo> > sourceStationData = new List <IMongoCollection <RecordMongo> >();
foreach (IMongoCollection <RecordMongo> sd in stationData)
{
pieces = sd.CollectionNamespace.CollectionName.Split('_');
if (pieces[4] == vcode)
{
sourceStationData.Add(sd);
}
}
foreach (RecordMongo r in v.records)
{
//synth year is local time
universal = r.time.ToUniversalTime();
int h = universal.Hour;
int doy = universal.DayOfYear;
hourofsyntheticyear++;
int foundValues = 0;
List <double> valuesForHour = new List <double>();
foreach (IMongoCollection <RecordMongo> sd in sourceStationData)
{
pieces = sd.CollectionNamespace.CollectionName.Split('_');
string source = pieces[2];
if (source.Contains("NOAA"))
{
source = "NOAA";
}
else
{
source = "IDEAM";
}
vm = AnnualSummary.getVariableMetaFromDB(vcode, source, db);
var project =
BsonDocument.Parse(
"{value: '$value',time:'$time',dayOfYear: {$dayOfYear: '$time'},hour: {$hour: '$time'}}");
try
{
var aggregationDocument =
sd.Aggregate()
.Unwind("value")
.Project(project)
.Match(BsonDocument.Parse("{$and:[" +
"{'dayOfYear' : {$eq : " + doy.ToString() + "}}" +
",{'hour' : {$eq : " + h.ToString() + "}}" +
",{'value':{$lte:" + vm.max.ToString() + " }}" +
",{'value':{$gte:" + vm.min.ToString() + "}}]}"))
.ToList();
IEnumerable <IGrouping <int, BsonDocument> > query = aggregationDocument.GroupBy(
doc => doc.GetValue("dayOfYear").ToInt32(),
doc => doc);
foreach (IGrouping <int, BsonDocument> hourValsGroup in query)
{
foreach (BsonDocument name in hourValsGroup)
{
double value = name.GetValue("value").ToDouble();
foundValues++;
//check nub and HRs are in the right range
if (vcode == "HR" && value <= 1)
{
value = value * 100;
}
if (vcode == "NUB")
{
//noaa's cloud is oktas
if (value == 9)
{
value = 10;
}
else
{
value = (int)(value / 8.0 * 10);
}
}
valuesForHour.Add(value);
}
}
}
catch (Exception e)
{
this.addLineToLogFile("WARN: " + synthYear.name + vcode + " error finding hourly values at day of year: " + doy + " ,hour : " + h);
}
}
if (foundValues != 0)
{
if (meanMedian == "meanHour")
{
r.value = Accord.Statistics.Measures.Mean(valuesForHour.ToArray());
}
if (meanMedian == "medianHour")
{
r.value = Accord.Statistics.Measures.Median(valuesForHour.ToArray());
}
if (meanMedian == "randomHour")
{
int total = valuesForHour.Count;
Random rand = new Random();
r.value = valuesForHour[rand.Next(0, total)];
}
}
}
}
private async Task generateHour(string vcode, SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData, string meanMedian)
{
var v = synthYear.variables.Find(x => x.name == vcode);
var builder = Builders <RecordMongo> .Filter;
string[] pieces;
VariableMeta vm;
int hourofsyntheticyear = 0;
DateTime local = new DateTime();
DateTime universal = new DateTime();
//find collections with current variable
List <IMongoCollection <RecordMongo> > sourceStationData = new List <IMongoCollection <RecordMongo> >();
foreach (IMongoCollection <RecordMongo> sd in stationData)
{
pieces = sd.CollectionNamespace.CollectionName.Split('_');
if (pieces[4] == vcode)
{
sourceStationData.Add(sd);
}
}
foreach (RecordMongo r in v.records)
{
//this is the time we need to fill
//need to filter for month day and hour
int m = r.time.Month;
int d = r.time.Day;
int h = r.time.Hour;
hourofsyntheticyear++;
double value = 0;
int foundValues = 0;
List <double> valuesForHour = new List <double>();
foreach (IMongoCollection <RecordMongo> sd in sourceStationData)
{
//only if the vcode matches
pieces = sd.CollectionNamespace.CollectionName.Split('_');
string source = pieces[2];
vm = AnnualSummary.getVariableMetaFromDB(vcode, source, db);
int startYr = 0;
int endYr = 0;
getFirstLastYear(sd, ref startYr, ref endYr);
if (startYr == 1)
{
startYr = 1980;
}
if (endYr == 1)
{
startYr = 2018;
}
for (int y = startYr; y < endYr; y++)
{
local = new DateTime(y, m, d, h, 0, 0);
universal = local.ToUniversalTime();
var filter = builder.Eq("time", universal) & builder.Gte("value", vm.min) & builder.Lte("value", vm.max);
//some collections have duplicate timestamps!
using (IAsyncCursor <RecordMongo> cursor = await sd.FindAsync(filter))
{
while (await cursor.MoveNextAsync())
{
IEnumerable <RecordMongo> documents = cursor.Current;
//insert into the station collection
foreach (RecordMongo sdrm in documents)
{
value = sdrm.value;
if (vcode == "HR" && sdrm.value <= 1)
{
value = value * 100;
}
if (vcode == "NUB")
{
if (value == 9)
{
value = 10;
}
value = (int)(value / 8.0 * 10);
}
valuesForHour.Add(value);
foundValues++;
}
}
}
}
}
if (foundValues != 0)
{
if (meanMedian == "meanHour")
{
r.value = Accord.Statistics.Measures.Mean(valuesForHour.ToArray());
}
if (meanMedian == "medianHour")
{
r.value = Accord.Statistics.Measures.Median(valuesForHour.ToArray());
}
if (meanMedian == "randomHour")
{
int total = valuesForHour.Count;
Random rand = new Random();
r.value = valuesForHour[rand.Next(0, total)];
}
}
}
}
