private CollectionMongo generateVariableYear(string name)
{
//one collection per variable
CollectionMongo cm = new CollectionMongo();
cm.name = name;
//loop all hours in 8760 add a new RecordMongo
int[] daysInMonths = new int[] { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 };
int month = 0;
int year = 2001;
foreach (int m in daysInMonths)
{
month++;
for (int d = 1; d <= m; d++)
{
for (int h = 0; h < 24; h++)
{
RecordMongo rm = new RecordMongo();
DateTime dt = new DateTime(year, month, d, h, 0, 0);
rm.value = -999.9;
rm.time = dt;
rm.processNote = "ccSyntheticYear";
cm.records.Add(rm);
}
}
}
return(cm);
}
private void addValuesToSynthYear(List <RecordMongo> candidateDay, ref CollectionMongo variableRecords, int doy, string vcode)
{
try
{
//assign to synthetic year
List <RecordMongo> synthDay = variableRecords.records.FindAll(x => x.time.DayOfYear == doy);
if (synthDay.Count > 0)
{
foreach (RecordMongo cm in candidateDay)
{
double value = cm.value;
RecordMongo synthRecordForUpdate = synthDay.Find(x => x.time.Hour == cm.time.Hour);
synthRecordForUpdate.value = value;
}
}
}
catch (Exception e)
{
this.addLineToLogFile("WARN: " + vcode + " error adding cdf day on day of year: " + doy);
}
}
private void addRecord(RecordMongo rm, string cName)
{
var cm = newCleanData.Find(x => x.name == cName);
cm.records.Add(rm);
}
private async Task selectDayOfYearCDF(string vcode, SyntheticYear synthYear, List <IMongoCollection <RecordMongo> > stationData)
{
var v = synthYear.variables.Find(x => x.name == vcode);
var builder = Builders <RecordMongo> .Filter;
string[] pieces;
VariableMeta vm;
List <int> missingDays = new List <int>();
List <List <RecordMongo> > possDayValues;
//find collections with current variable
List <IMongoCollection <RecordMongo> > sourceStationData = new List <IMongoCollection <RecordMongo> >();
foreach (IMongoCollection <RecordMongo> sd in stationData)
{
pieces = sd.CollectionNamespace.CollectionName.Split('_');
if (pieces[4] == vcode)
{
sourceStationData.Add(sd);
}
}
for (int doy = 1; doy < 366; doy++)
{
//each day will have several canadidate days sourced from each collection of same variable
possDayValues = new List <List <RecordMongo> >();
foreach (IMongoCollection <RecordMongo> sd in sourceStationData)
{
pieces = sd.CollectionNamespace.CollectionName.Split('_');
string source = pieces[2];
if (source.Contains("NOAA"))
{
source = "NOAA";
}
else
{
source = "IDEAM";
}
vm = AnnualSummary.getVariableMetaFromDB(vcode, source, db);
var project =
BsonDocument.Parse(
"{value: '$value',time:'$time',dayOfYear: {$dayOfYear: '$time'},year: {$year: '$time'}}");
try
{
//.Match(BsonDocument.Parse("{'dayOfYear' : {$eq : " + doy.ToString() + "}}"))
var aggregationDocument =
sd.Aggregate()
.Unwind("value")
.Project(project)
.Match(BsonDocument.Parse("{$and:[" +
"{'dayOfYear' : {$eq : " + doy.ToString() + "}}" +
",{'value':{$lte:" + vm.max.ToString() + " }}" +
",{'value':{$gte:" + vm.min.ToString() + "}}]}"))
.ToList();
IEnumerable <IGrouping <int, BsonDocument> > query = aggregationDocument.GroupBy(
doc => doc.GetValue("year").ToInt32(),
doc => doc);
foreach (IGrouping <int, BsonDocument> yearDayGroup in query)
{
var year = yearDayGroup.Key;
var hours = yearDayGroup.Count();
//one group per day per year count should be 24
//but many noaa data are sometimes day time only 6-6 12 readings
if (hours >= 12)
{
List <RecordMongo> dayValues = new List <RecordMongo>();
foreach (BsonDocument name in yearDayGroup)
{
RecordMongo rm = new RecordMongo();
double value = name.GetValue("value").ToDouble();
//check nub and HRs are in the right range
if (vcode == "HR" && value <= 1)
{
value = value * 100;
}
if (vcode == "NUB")
{
//noaa's cloud is oktas
if (value == 9)
{
value = 10;
}
else
{
value = (int)(value / 8.0 * 10);
}
}
rm.value = value;
rm.time = name.GetValue("time").ToLocalTime();
dayValues.Add(rm);
}
possDayValues.Add(dayValues);
}
}
}
catch (Exception e)
{
this.addLineToLogFile("WARN: " + synthYear.name + vcode + " error finding cdf day at day of year: " + doy);
}
}
if (possDayValues.Count > 0)
{
List <RecordMongo> dayToInsert = typicalDay(possDayValues, vcode);
addValuesToSynthYear(dayToInsert, ref v, doy, vcode);
}
else
{
//no possible days found empty day
missingDays.Add(doy);
}
}
if (missingDays.Count > 0)
{
fillMissingDays(missingDays);
}
//fill missing days
}
public void insertManyRecord(string collectionName, string file, char split)
{
var collection = db.GetCollection <RecordMongo>(collectionName);
var listOfDocuments = new List <RecordMongo>();
var limitAtOnce = 1000;
var current = 0;
StreamReader read = new StreamReader(file);
string line = read.ReadLine();
string[] prts;
DateTime dt = new DateTime();
while (line != null)
{
prts = line.Split(split);
if (!DateTime.TryParse(prts[1], out dt))
{
var b = 0;
}
var dataToInsert = new RecordMongo
{
stationCode = Convert.ToInt32(prts[0]),
time = dt,
value = Convert.ToDouble(prts[2])
};
if (collectionName.Contains("NOAA"))
{
//noaa uses UTC so this subtracts 5 hours to bogota
dataToInsert.time = dataToInsert.time.ToLocalTime();
}
else
{
if (collectionName.Contains("variable"))
{
//ideams date stamp is local -5!!!!
dataToInsert.time = dataToInsert.time.AddHours(5);
//set as local
dataToInsert.time = DateTime.SpecifyKind(dataToInsert.time, DateTimeKind.Local);
}
else
{
dataToInsert.time = DateTime.SpecifyKind(dataToInsert.time, DateTimeKind.Local);
}
}
listOfDocuments.Add(dataToInsert);
if (++current == limitAtOnce)
{
current = 0;
var listToInsert = listOfDocuments;
var t = new Task(() => { collection.InsertManyAsync(listToInsert); });
t.Start();
listOfDocuments = new List <RecordMongo>();
}
line = read.ReadLine();
}
// insert remainder
//await collection.InsertManyAsync(listOfDocuments);
var f = new Task(() => { collection.InsertManyAsync(listOfDocuments); });
f.Start();
read.Close();
}
public async Task sortByDateAndAverage(int scode, string collname, VariableMeta vm, string newcollectionname)
{
IMongoCollection <RecordMongo> collection = db.GetCollection <RecordMongo>(collname);
var filter = FilterDefinition <RecordMongo> .Empty;
var sorter = Builders <RecordMongo> .Sort.Ascending("time");
FindOptions <RecordMongo> options = new FindOptions <RecordMongo>
{
BatchSize = 500,
NoCursorTimeout = false,
Sort = sorter
};
DateTime currentHour = new DateTime();
bool firstrecord = true;
using (IAsyncCursor <RecordMongo> cursor = await collection.FindAsync(filter, options))
{
double hourtotal = 0;
int recordsPerHr = 0;
while (await cursor.MoveNextAsync())
{
IEnumerable <RecordMongo> batch = cursor.Current;
foreach (RecordMongo rm in batch)
{
//only if the value is in range
if (rm.value > vm.min && rm.value < vm.max)
{
if (firstrecord)
{
currentHour = rm.time;
firstrecord = false;
}
if (rm.time.DayOfYear == currentHour.DayOfYear && rm.time.Hour == currentHour.Hour)
{
recordsPerHr++;
hourtotal += rm.value;
}
else
{
//make a new record and add to the list
RecordMongo avrm = new RecordMongo();
avrm.processNote = "averaged from 10min readings";
avrm.stationCode = scode;
avrm.time = new DateTime(currentHour.Year, currentHour.Month, currentHour.Day, currentHour.Hour, 0, 0);
avrm.value = hourtotal / recordsPerHr;
if (!Double.IsNaN(avrm.value))
{
addRecord(avrm, newcollectionname);
}
//reset the counter and total
recordsPerHr = 0;
hourtotal = 0;
//set the new hour
currentHour = rm.time;
}
}
}
}
}
}