public async Task syntheticYearBatchFixer(string method)
{
CityYearFixer cyf = new CityYearFixer();
List <NeededData> neededData = cyf.readRequiredData();
stations = StationGrouping.getAllStationsFromDB(db);
var coll = db.GetCollection <StationGroup>("cityRegionGroups");
var allCityGroups = coll.Find(FilterDefinition <StationGroup> .Empty).ToList();
this.addLineToLogFile("INFO: starting batch of synth years");
List <Task> tasks = new List <Task>();
foreach (StationGroup sg in allCityGroups)
{
var city = sg.name;
NeededData cityNeeds = neededData.Find(nd => nd.name == city);
if (cityNeeds != null)
{
if (city != "MITU")
{
var cityGroup = allCityGroups.Find(x => x.name == city);
var stationCollNames = getStationsColNames(cityGroup);
tasks.Add(fixCity(city, method, cityNeeds, stationCollNames));
}
}
}
await Task.WhenAll(tasks);
}
public async Task fixCity(string city, string method, NeededData cityNeeds, List <string> stationCollNames)
{
List <IMongoCollection <RecordMongo> > stationData = new List <IMongoCollection <RecordMongo> >();
try
{
//ignore 10min collections
stationData = getTheStationData(stationCollNames.FindAll(s => s.Contains("_60")));
this.addLineToLogFile("INFO: found ref data for " + city + " synth year");
}
catch
{
this.addLineToLogFile("WARN: no ref data found for " + city + " synth year");
}
//read the synthyear for this city
var collection = db.GetCollection <SyntheticYear>(city + "_medianHour");
List <SyntheticYear> synthYear = collection.Find(FilterDefinition <SyntheticYear> .Empty).ToList();
SyntheticYear sy = synthYear[0];
SyntheticYear.convertSyntheticYear(ref sy);
try
{
await getDaysForSelectedVariables(sy, stationData, method, cityNeeds.reqVariables);
this.addLineToLogFile("INFO: calculated data for " + city + " synth year");
}
catch
{
this.addLineToLogFile("WARN: error in calculating values for " + city + " synth year");
}
try
{
holeFiller(holeFinder(ref sy), ref sy);
this.addLineToLogFile("INFO: hole filling succeeded for " + city + " synth year");
}
catch
{
this.addLineToLogFile("WARN: hole filling failed for " + city + " synth year");
}
try
{
insertSytheticYear(city + "_" + method + "regionFix", sy);
this.addLineToLogFile("INFO: " + city + " synth year was stored in DB");
}
catch
{
this.addLineToLogFile("WARN: " + city + " synth year was not stored in DB");
}
}
public List <NeededData> readRequiredData()
{
List <NeededData> neededData = new List <NeededData>();
StreamReader sr = new StreamReader(@"C:\Users\Admin\Documents\projects\IAPP\piloto\Climate\ClimateDataETL\needed.csv");
string line = sr.ReadLine();
while (line != null)
{
string[] parts = line.Split(',');
var nd = new NeededData();
nd.name = parts[0];
for (int i = 1; i < parts.Length; i++)
{
if (parts[i] != "")
{
nd.reqVariables.Add(parts[i]);
}
}
neededData.Add(nd);
line = sr.ReadLine();
}
sr.Close();
return(neededData);
}
