public void insertSytheticYear(string collectionName, SyntheticYear synthYear)
{
var collection = db.GetCollection <SyntheticYear>(collectionName);
var f = new Task(() => { collection.InsertOneAsync(synthYear); });
f.Start();
}
private void holeFiller(List <Hole> allHoles, ref SyntheticYear synthYear)
{
DateTime start = new DateTime();
DateTime end = new DateTime();
DateTime current = new DateTime();
foreach (Hole h in allHoles)
{
TimeSpan holesize = h.getHoleEnd() - h.getHoleStart();
if (holesize.Hours < 5)
{
//check for start end year
try
{
if (h.getHoleStart().Month == 1 && h.getHoleStart().Day == 1 && h.getHoleStart().Hour == 0)
{
//hole at first hour of year
start = new DateTime(h.getHoleStart().Year, 12, 31, 23, 0, 0);
}
else
{
start = h.getHoleStart().AddHours(-1);
}
if (h.getHoleEnd().Month == 12 && h.getHoleEnd().Day == 31 && h.getHoleEnd().Hour == 23)
{
//hole at final hour of year
end = new DateTime(h.getHoleStart().Year, 1, 1, 0, 0, 0);
}
else
{
end = h.getHoleEnd().AddHours(1);
}
//interpolation
double v1 = synthYear.variables.Find(x => x.name == h.vcode).records.Find(r => r.time == start).value;
double v2 = synthYear.variables.Find(x => x.name == h.vcode).records.Find(r => r.time == end).value;
double range = v2 - v1;
double inc = range / (holesize.Hours + 2);
for (int i = 1; i <= holesize.Hours + 1; i++)
{
current = start.AddHours(i);
var tofill = synthYear.variables.Find(x => x.name == h.vcode).records.Find(r => r.time == current);
tofill.value = v1 + (i * inc);
}
}
catch (Exception e)
{
this.addLineToLogFile("WARN: " + h.vcode + " holefilling error between " + h.getHoleStart().ToString() + " and " + h.getHoleEnd().ToString());
}
}
else
{
//select from neighour hood of days in week
}
}
}
public static void convertSyntheticYear(ref SyntheticYear yearToCopy)
{
//input synth year is in UTC time
foreach (CollectionMongo vcoll in yearToCopy.variables)
{
foreach (RecordMongo r in vcoll.records)
{
r.time = r.time.ToLocalTime();
}
}
}
//this is for wea to epw
public EPWWriter(SyntheticYear locdata, string d)
{
this.synthYear = locdata;
this.directory = d;
this.city = this.synthYear.name;
this.country = "Colombia";// StringTools.stripExt(new string(this.ld.where));
Directory.CreateDirectory(this.directory + "\\epw");
this.epw = new StreamWriter(this.directory + "\\epw\\" + city + ".epw");
writeEPW();
this.epw.Close();
}
public async Task makeSynthYear(StationGroup sg, string method)
{
List <IMongoCollection <RecordMongo> > stationData = new List <IMongoCollection <RecordMongo> >();
try
{
var stationCollNames = getStationsColNames(sg);
await index60Minutes(stationCollNames);
//ignore 10min collections
stationData = getTheStationData(stationCollNames.FindAll(s => s.Contains("60")));
this.addLineToLogFile("INFO: found ref data for " + sg.name + " synth year");
}
catch
{
this.addLineToLogFile("WARN: no ref data found for " + sg.name + " synth year");
}
SyntheticYear synthYear = new SyntheticYear();
synthYear.name = sg.name;
try
{
await getDaysForVariables(synthYear, stationData, method);
this.addLineToLogFile("INFO: calculated data for " + sg.name + " synth year");
}
catch
{
this.addLineToLogFile("WARN: error in calculating values for " + sg.name + " synth year");
}
try
{
holeFiller(holeFinder(ref synthYear), ref synthYear);
this.addLineToLogFile("INFO: hole filling succeeded for " + sg.name + " synth year");
}
catch
{
this.addLineToLogFile("WARN: hole filling failed for " + sg.name + " synth year");
}
try
{
insertSytheticYear(sg.name + "_" + method, synthYear);
this.addLineToLogFile("INFO: " + sg.name + " synth year was stored in DB");
}
catch
{
this.addLineToLogFile("WARN: " + sg.name + " synth year was not stored in DB");
}
}
public async Task fixCity(string city, string method, NeededData cityNeeds, List <string> stationCollNames)
{
List <IMongoCollection <RecordMongo> > stationData = new List <IMongoCollection <RecordMongo> >();
try
{
//ignore 10min collections
stationData = getTheStationData(stationCollNames.FindAll(s => s.Contains("_60")));
this.addLineToLogFile("INFO: found ref data for " + city + " synth year");
}
catch
{
this.addLineToLogFile("WARN: no ref data found for " + city + " synth year");
}
//read the synthyear for this city
var collection = db.GetCollection <SyntheticYear>(city + "_medianHour");
List <SyntheticYear> synthYear = collection.Find(FilterDefinition <SyntheticYear> .Empty).ToList();
SyntheticYear sy = synthYear[0];
SyntheticYear.convertSyntheticYear(ref sy);
try
{
await getDaysForSelectedVariables(sy, stationData, method, cityNeeds.reqVariables);
this.addLineToLogFile("INFO: calculated data for " + city + " synth year");
}
catch
{
this.addLineToLogFile("WARN: error in calculating values for " + city + " synth year");
}
try
{
holeFiller(holeFinder(ref sy), ref sy);
this.addLineToLogFile("INFO: hole filling succeeded for " + city + " synth year");
}
catch
{
this.addLineToLogFile("WARN: hole filling failed for " + city + " synth year");
}
try
{
insertSytheticYear(city + "_" + method + "regionFix", sy);
this.addLineToLogFile("INFO: " + city + " synth year was stored in DB");
}
catch
{
this.addLineToLogFile("WARN: " + city + " synth year was not stored in DB");
}
}
private async Task getDaysForVariables(SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData, string method)
{
if (method == "cdfDay")
{
await Task.WhenAll(synthYear.variables.Select(c => selectDayOfYearCDF(c.name, synthYear, stationData)));
this.addLineToLogFile("INFO: got CDF daily values for " + synthYear.name + " synth year");
}
else
{
await Task.WhenAll(synthYear.variables.Select(c => generateHour(c.name, synthYear, stationData, method)));
this.addLineToLogFile("INFO: calculated hourly values for " + synthYear.name + " synth year");
}
}
private List <Hole> holeFinder(ref SyntheticYear synthYear)
{
//fix night radiaiton
var rs = synthYear.variables.Find(x => x.name == "RS");
nightRadiation(ref rs);
List <Hole> allHoles = new List <Hole>();
foreach (CollectionMongo c in synthYear.variables)
{
bool newhole = true;
Hole h = new Hole();
DateTime prevDT = new DateTime();
foreach (RecordMongo rm in c.records)
{
if (rm.value == -999.9)
{
if (newhole)
{
//found first of new hole
h = new Hole();
h.vcode = c.name;
h.setHoleStart(rm.time);
newhole = false;
}
}
if (rm.value != -999.9 && !newhole)
{
//end of hole
h.setHoleEnd(prevDT);
newhole = true;
allHoles.Add(h);
}
prevDT = rm.time;
}
//hole at end of year
if (!newhole)
{
h.setHoleEnd(prevDT);
newhole = true;
allHoles.Add(h);
}
}
return(allHoles);
}
private async Task getDaysForSelectedVariables(SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData, string method, List <string> fields)
{
if (method == "cdfDay")
{
List <Task> tasks = new List <Task>();
foreach (string field in fields)
{
tasks.Add(selectDayOfYearCDF(field, synthYear, stationData));
}
await Task.WhenAll(tasks);
this.addLineToLogFile("INFO: got CDF daily values for " + synthYear.name + " synth year");
}
else
{
//List<Task> tasks = new List<Task>();
foreach (string field in fields)
{
await generateHour(field, synthYear, stationData, method);
}
//await Task.WhenAll(tasks);
this.addLineToLogFile("INFO: calculating hourly values for " + synthYear.name + " synth year");
}
}
private async Task selectDayOfYearCDF(string vcode, SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData)
{
var v = synthYear.variables.Find(x => x.name == vcode);
var builder = Builders <RecordMongo> .Filter;
string[] pieces;
VariableMeta vm;
List <int> missingDays = new List <int>();
List <List <RecordMongo> > possDayValues;
//find collections with current variable
List <IMongoCollection <RecordMongo> > sourceStationData = new List <IMongoCollection <RecordMongo> >();
foreach (IMongoCollection <RecordMongo> sd in stationData)
{
pieces = sd.CollectionNamespace.CollectionName.Split('_');
if (pieces[4] == vcode)
{
sourceStationData.Add(sd);
}
}
for (int doy = 1; doy < 366; doy++)
{
//each day will have several canadidate days sourced from each collection of same variable
possDayValues = new List <List <RecordMongo> >();
foreach (IMongoCollection <RecordMongo> sd in sourceStationData)
{
pieces = sd.CollectionNamespace.CollectionName.Split('_');
string source = pieces[2];
if (source.Contains("NOAA"))
{
source = "NOAA";
}
else
{
source = "IDEAM";
}
vm = AnnualSummary.getVariableMetaFromDB(vcode, source, db);
var project =
BsonDocument.Parse(
"{value: '$value',time:'$time',dayOfYear: {$dayOfYear: '$time'},year: {$year: '$time'}}");
try
{
//.Match(BsonDocument.Parse("{'dayOfYear' : {$eq : " + doy.ToString() + "}}"))
var aggregationDocument =
sd.Aggregate()
.Unwind("value")
.Project(project)
.Match(BsonDocument.Parse("{$and:[" +
"{'dayOfYear' : {$eq : " + doy.ToString() + "}}" +
",{'value':{$lte:" + vm.max.ToString() + " }}" +
",{'value':{$gte:" + vm.min.ToString() + "}}]}"))
.ToList();
IEnumerable <IGrouping <int, BsonDocument> > query = aggregationDocument.GroupBy(
doc => doc.GetValue("year").ToInt32(),
doc => doc);
foreach (IGrouping <int, BsonDocument> yearDayGroup in query)
{
var year = yearDayGroup.Key;
var hours = yearDayGroup.Count();
//one group per day per year count should be 24
//but many noaa data are sometimes day time only 6-6 12 readings
if (hours >= 12)
{
List <RecordMongo> dayValues = new List <RecordMongo>();
foreach (BsonDocument name in yearDayGroup)
{
RecordMongo rm = new RecordMongo();
double value = name.GetValue("value").ToDouble();
//check nub and HRs are in the right range
if (vcode == "HR" && value <= 1)
{
value = value * 100;
}
if (vcode == "NUB")
{
//noaa's cloud is oktas
if (value == 9)
{
value = 10;
}
else
{
value = (int)(value / 8.0 * 10);
}
}
rm.value = value;
rm.time = name.GetValue("time").ToLocalTime();
dayValues.Add(rm);
}
possDayValues.Add(dayValues);
}
}
}
catch (Exception e)
{
this.addLineToLogFile("WARN: " + synthYear.name + vcode + " error finding cdf day at day of year: " + doy);
}
}
if (possDayValues.Count > 0)
{
List <RecordMongo> dayToInsert = typicalDay(possDayValues, vcode);
addValuesToSynthYear(dayToInsert, ref v, doy, vcode);
}
else
{
//no possible days found empty day
missingDays.Add(doy);
}
}
if (missingDays.Count > 0)
{
fillMissingDays(missingDays);
}
//fill missing days
}
private async Task generateHour2(string vcode, SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData, string meanMedian)
{
var v = synthYear.variables.Find(x => x.name == vcode);
var builder = Builders <RecordMongo> .Filter;
string[] pieces;
VariableMeta vm;
int hourofsyntheticyear = 0;
DateTime universal = new DateTime();
//find collections with current variable
List <IMongoCollection <RecordMongo> > sourceStationData = new List <IMongoCollection <RecordMongo> >();
foreach (IMongoCollection <RecordMongo> sd in stationData)
{
pieces = sd.CollectionNamespace.CollectionName.Split('_');
if (pieces[4] == vcode)
{
sourceStationData.Add(sd);
}
}
foreach (RecordMongo r in v.records)
{
//synth year is local time
universal = r.time.ToUniversalTime();
int h = universal.Hour;
int doy = universal.DayOfYear;
hourofsyntheticyear++;
int foundValues = 0;
List <double> valuesForHour = new List <double>();
foreach (IMongoCollection <RecordMongo> sd in sourceStationData)
{
pieces = sd.CollectionNamespace.CollectionName.Split('_');
string source = pieces[2];
if (source.Contains("NOAA"))
{
source = "NOAA";
}
else
{
source = "IDEAM";
}
vm = AnnualSummary.getVariableMetaFromDB(vcode, source, db);
var project =
BsonDocument.Parse(
"{value: '$value',time:'$time',dayOfYear: {$dayOfYear: '$time'},hour: {$hour: '$time'}}");
try
{
var aggregationDocument =
sd.Aggregate()
.Unwind("value")
.Project(project)
.Match(BsonDocument.Parse("{$and:[" +
"{'dayOfYear' : {$eq : " + doy.ToString() + "}}" +
",{'hour' : {$eq : " + h.ToString() + "}}" +
",{'value':{$lte:" + vm.max.ToString() + " }}" +
",{'value':{$gte:" + vm.min.ToString() + "}}]}"))
.ToList();
IEnumerable <IGrouping <int, BsonDocument> > query = aggregationDocument.GroupBy(
doc => doc.GetValue("dayOfYear").ToInt32(),
doc => doc);
foreach (IGrouping <int, BsonDocument> hourValsGroup in query)
{
foreach (BsonDocument name in hourValsGroup)
{
double value = name.GetValue("value").ToDouble();
foundValues++;
//check nub and HRs are in the right range
if (vcode == "HR" && value <= 1)
{
value = value * 100;
}
if (vcode == "NUB")
{
//noaa's cloud is oktas
if (value == 9)
{
value = 10;
}
else
{
value = (int)(value / 8.0 * 10);
}
}
valuesForHour.Add(value);
}
}
}
catch (Exception e)
{
this.addLineToLogFile("WARN: " + synthYear.name + vcode + " error finding hourly values at day of year: " + doy + " ,hour : " + h);
}
}
if (foundValues != 0)
{
if (meanMedian == "meanHour")
{
r.value = Accord.Statistics.Measures.Mean(valuesForHour.ToArray());
}
if (meanMedian == "medianHour")
{
r.value = Accord.Statistics.Measures.Median(valuesForHour.ToArray());
}
if (meanMedian == "randomHour")
{
int total = valuesForHour.Count;
Random rand = new Random();
r.value = valuesForHour[rand.Next(0, total)];
}
}
}
}
private async Task generateHour(string vcode, SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData, string meanMedian)
{
var v = synthYear.variables.Find(x => x.name == vcode);
var builder = Builders <RecordMongo> .Filter;
string[] pieces;
VariableMeta vm;
int hourofsyntheticyear = 0;
DateTime local = new DateTime();
DateTime universal = new DateTime();
//find collections with current variable
List <IMongoCollection <RecordMongo> > sourceStationData = new List <IMongoCollection <RecordMongo> >();
foreach (IMongoCollection <RecordMongo> sd in stationData)
{
pieces = sd.CollectionNamespace.CollectionName.Split('_');
if (pieces[4] == vcode)
{
sourceStationData.Add(sd);
}
}
foreach (RecordMongo r in v.records)
{
//this is the time we need to fill
//need to filter for month day and hour
int m = r.time.Month;
int d = r.time.Day;
int h = r.time.Hour;
hourofsyntheticyear++;
double value = 0;
int foundValues = 0;
List <double> valuesForHour = new List <double>();
foreach (IMongoCollection <RecordMongo> sd in sourceStationData)
{
//only if the vcode matches
pieces = sd.CollectionNamespace.CollectionName.Split('_');
string source = pieces[2];
vm = AnnualSummary.getVariableMetaFromDB(vcode, source, db);
int startYr = 0;
int endYr = 0;
getFirstLastYear(sd, ref startYr, ref endYr);
if (startYr == 1)
{
startYr = 1980;
}
if (endYr == 1)
{
startYr = 2018;
}
for (int y = startYr; y < endYr; y++)
{
local = new DateTime(y, m, d, h, 0, 0);
universal = local.ToUniversalTime();
var filter = builder.Eq("time", universal) & builder.Gte("value", vm.min) & builder.Lte("value", vm.max);
//some collections have duplicate timestamps!
using (IAsyncCursor <RecordMongo> cursor = await sd.FindAsync(filter))
{
while (await cursor.MoveNextAsync())
{
IEnumerable <RecordMongo> documents = cursor.Current;
//insert into the station collection
foreach (RecordMongo sdrm in documents)
{
value = sdrm.value;
if (vcode == "HR" && sdrm.value <= 1)
{
value = value * 100;
}
if (vcode == "NUB")
{
if (value == 9)
{
value = 10;
}
value = (int)(value / 8.0 * 10);
}
valuesForHour.Add(value);
foundValues++;
}
}
}
}
}
if (foundValues != 0)
{
if (meanMedian == "meanHour")
{
r.value = Accord.Statistics.Measures.Mean(valuesForHour.ToArray());
}
if (meanMedian == "medianHour")
{
r.value = Accord.Statistics.Measures.Median(valuesForHour.ToArray());
}
if (meanMedian == "randomHour")
{
int total = valuesForHour.Count;
Random rand = new Random();
r.value = valuesForHour[rand.Next(0, total)];
}
}
}
}
