private async Task splitter(string collectionname)
{
List <int> codes = MongoTools.distinct(collectionname, db, "stationCode");
IMongoCollection <BsonDocument> variableCollection = db.GetCollection <BsonDocument>(collectionname);
FindOptions <BsonDocument> options = new FindOptions <BsonDocument>
{
BatchSize = 1000,
NoCursorTimeout = false
};
foreach (int stationcode in codes)
{
//get or make a station collection
IMongoCollection <BsonDocument> stationVariableCollection = db.GetCollection <BsonDocument>("s_" + stationcode + "_" + collectionname);
var builder = Builders <BsonDocument> .Filter;
var filter = builder.Eq("stationCode", stationcode);
//find in the variable collection
using (IAsyncCursor <BsonDocument> cursor = await variableCollection.FindAsync(filter, options))
{
while (await cursor.MoveNextAsync())
{
IEnumerable <BsonDocument> documents = cursor.Current;
//insert into the station collection
await stationVariableCollection.InsertManyAsync(documents);
}
}
}
}
public void outputAnnual()
{
//this could select the most recent annual
var names = MongoTools.collectionNames(db);
var summ = names.FindAll(c => c.Contains("annualStationSummary"));
var coll = db.GetCollection <StationSummary>("annualStationSummary_2019_6_17_17_16_41");
List <StationSummary> stations = coll.Find(FilterDefinition <StationSummary> .Empty).ToList();
StreamWriter sw = new StreamWriter("annualSummary.csv");
sw.WriteLine("code,name,source,TS,HR,PR,VV,DV,NUB");
foreach (StationSummary ss in stations)
{
Station s = getStationFromMongo(ss.code, db);
sw.WriteLine(string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8}",
ss.code, s.name, s.source,
ss.getTotalYrs("TS").ToString(),
ss.getTotalYrs("HR").ToString(),
ss.getTotalYrs("PR").ToString(),
ss.getTotalYrs("VV").ToString(),
ss.getTotalYrs("DV").ToString(),
ss.getTotalYrs("NUB").ToString()
));
}
sw.Close();
}
public void insertManyMonthlySummary()
{
DateTime dt = DateTime.Now;
string collectionname = "monthlyStationSummary_" + dt.Year.ToString()
+ "_" + dt.Month.ToString() + "_" + dt.Day.ToString() + "_" + dt.Hour.ToString() + "_" + dt.Minute.ToString() + "_" + dt.Second.ToString();
var collection = db.GetCollection <StationMonthly>(collectionname);
MongoTools.storeSummaryCollectionName(db, collectionname);
var listOfDocuments = new List <StationMonthly>();
var limitAtOnce = 1000;
var current = 0;
foreach (StationMonthly ss in stations)
{
listOfDocuments.Add(ss);
if (++current == limitAtOnce)
{
current = 0;
var listToInsert = listOfDocuments;
var t = new Task(() => { collection.InsertManyAsync(listToInsert); });
t.Start();
listOfDocuments = new List <StationMonthly>();
}
}
var f = new Task(() => { collection.InsertManyAsync(listOfDocuments); });
f.Start();
}
public void cleanUp()
{
List <string> collNames = MongoTools.collectionNames(db);
string vname = "";
int stationcode = 0;
string source = "";
int freq = 0;
foreach (string collection in collNames)
{
//all station record collections start with an s_
if (collection[0] == 's')
{
string[] parts = collection.Split('_');
stationcode = Convert.ToInt32(parts[1]);
vname = parts[4];
if (vname == "PA")
{
continue;
}
source = parts[2];
freq = Convert.ToInt32(parts[5]);
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
if (freq == 60)
{
string newname = convertNameToClean(collection);
CollectionMongo cm = new CollectionMongo();
cm.name = newname;
removeRecordsOutsideRange(stationcode, collection, meta, newname);
}
}
}
}
private List <string> getStationsColNames(StationGroup cityGroup)
{
List <string> collections = MongoTools.collectionNames(db);
string vname = "";
int scode = 0;
string source = "";
int freq = 0;
List <string> stationCollections = new List <string>();
foreach (string col in collections)
{
if (col[0] == 's')
{
string[] parts = col.Split('_');
try {
scode = Convert.ToInt32(parts[1]);
vname = parts[4];
source = parts[2];
freq = Convert.ToInt32(parts[5]);
}
catch (Exception e)
{
this.addLineToLogFile("WARN: error finding collection: " + col + "for city group:" + cityGroup.name);
}
foreach (int code in cityGroup.stationcodes)
{
if (scode == code)
{
stationCollections.Add(col);
}
}
}
}
return(stationCollections);
}
private List <RecordMongo> getACollection(int stationcode, string variable)
{
//get the collection with summary names
List <RecordMongo> records = new List <RecordMongo>();
var allCollections = MongoTools.collectionNames(db);
string vname = "";
int scode = 0;
string source = "";
int freq = 0;
int count = 0;
foreach (string collection in allCollections)
{
if (collection[0] == 's')
{
string[] parts = collection.Split('_');
scode = Convert.ToInt32(parts[1]);
vname = parts[4];
source = parts[2];
freq = Convert.ToInt32(parts[5]);
if (scode == stationcode && vname == variable && freq == 60)
{
IMongoCollection <RecordMongo> stationData = db.GetCollection <RecordMongo>(collection);
var filter = FilterDefinition <RecordMongo> .Empty;
records = stationData.Find(filter).ToList();
return(records);
}
}
}
return(records);
}
public void dateTimeTest(String source)
{
var names = MongoTools.collectionNames(db);
int count = 0;
List <string> subset = new List <string>();
foreach (String c in names)
{
if (c[0] == 's')
{
if (c.Contains(source))
{
subset.Add(c);
}
}
}
var wcss = splitList(subset, 10);
foreach (List <string> wcs in wcss)
{
testDaysGraphic(wcs, source + "p_" + count);
count++;
}
}
public void printCityRegionGroups()
{
db = MongoTools.connect("mongodb://localhost", "climaColombia");
var coll = db.GetCollection <StationGroup>("cityRegionGroups");
allRegionGroups = coll.Find(FilterDefinition <StationGroup> .Empty).ToList();
StreamWriter sw = new StreamWriter("regiongroups.csv");
foreach (StationGroup sg in allRegionGroups)
{
StringBuilder sb = new StringBuilder();
sb.Append(sg.name + ",");
foreach (int code in sg.stationcodes)
{
sb.Append(code + ",");
}
sw.WriteLine(sb.ToString());
}
sw.Close();
cities = MapTools.readCities();
stations = StationGrouping.getAllStationsFromDB(db);
JSONout.writeGroup(allRegionGroups, @"C:\Users\Admin\Documents\projects\IAPP\climaColombiaOrg\tools\cityGroups\cityregiongroups.json", stations, cities);
}
public async Task splitVariables(string dbname)
{
connect("mongodb://localhost", dbname);
List <string> collNames = MongoTools.collectionNames(db);
await splitLoop(collNames);
}
public static List <StationSummary> getCollectionAsList(string collectionname)
{
IMongoDatabase db = MongoTools.connect("mongodb://localhost", "climaColombia");
var coll = db.GetCollection <StationSummary>(collectionname);
var t = coll.Find(FilterDefinition <StationSummary> .Empty).ToList();
//db.DropCollection(collection);
return(t);
}
public void convert(string key)
{
//("mongodb://localhost/?maxPoolSize=555");
db = MongoTools.connect("mongodb://localhost/?maxPoolSize=1000", "climaColombia");
Task t1 = Task.Run(() => convert10min(key));
//add the processed data to mongo
//t1.Wait();
//foreach(CollectionMongo cm in newAveragedData)
//{
// insertMany(cm.records, cm.name);
//}
}
public void clean()
{
//("mongodb://localhost/?maxPoolSize=555");
db = MongoTools.connect("mongodb://localhost/?maxPoolSize=1000", "climaColombia");
Task t1 = Task.Run(() => cleanUp());
//add the processed data to mongo
t1.Wait();
foreach (CollectionMongo cm in newCleanData)
{
insertMany(cm.records, cm.name);
}
}
public static void checkRadiationLoaded()
{
IMongoDatabase db = MongoTools.connect("mongodb://localhost", "climaColombia");
//clean up bog buc with PA variable
List <string> collNames = MongoTools.collectionNames(db);
foreach (string collection in collNames)
{
if (collection.Contains("rad"))
{
var rad = 0;
}
}
}
public static void dropIndexes()
{
IMongoDatabase db = MongoTools.connect("mongodb://localhost", "climaColombia");
//clean up bog buc with PA variable
List <string> collNames = MongoTools.collectionNames(db);
foreach (string collection in collNames)
{
if (collection[0] == 's')
{
IMongoCollection <BsonDocument> coll = db.GetCollection <BsonDocument>(collection);
coll.Indexes.DropAllAsync();
}
}
}
public void graphAllTS_RS()
{
var colls = MongoTools.collectionNames(db);
foreach (string coll in colls)
{
if (coll[0] == 's')
{
if (coll.Contains("RS") || coll.Contains("TS"))
{
graphSingleStation(coll, coll);
}
}
}
}
public async Task splitByKeyWord(string dbname, string keyword)
{
connect("mongodb://localhost", dbname);
List <string> collNames = MongoTools.collectionNames(db);
List <string> subset = new List <string>();
foreach (String c in collNames)
{
if (c.Contains(keyword))
{
subset.Add(c);
}
}
await splitLoop(subset);
}
public static void cleanUpByKeyword(String key)
{
IMongoDatabase db = MongoTools.connect("mongodb://localhost", "climaColombia");
//clean up bog buc with PA variable
List <string> collNames = MongoTools.collectionNames(db);
foreach (string collection in collNames)
{
if (collection.Contains(key))
{
var coll = db.GetCollection <BsonDocument>(collection);
var t = coll.Find(new BsonDocument()).ToList();
db.DropCollection(collection);
}
}
}
private void getActiveStations()
{
List <string> collections = MongoTools.collectionNames(db);
foreach (string collection in collections)
{
if (collection[0] == 's')
{
string[] parts = collection.Split('_');
int code = Convert.ToInt32(parts[1]);
if (!activeStationCodes.Contains(code))
{
activeStationCodes.Add(code);
}
}
}
}
public void indexAll()
{
List <string> collNames = MongoTools.collectionNames(db);
string vname = "";
int stationcode = 0;
string source = "";
int freq = 0;
foreach (string collection in collNames)
{
//all station record collections start with an s_
if (collection[0] == 's')
{
createCollectionIndex(collection);
}
}
}
public StationGrouping(bool allideam)
{
db = MongoTools.connect("mongodb://localhost", "climaColombia");
getData();
if (allideam)
{
makeGroupsALLIDEAM();
}
else
{
makeGroups();
}
writeStationGroupIDs();
outputJSON();
storeInMongo();
writeStationCoords();
}
public async Task monthlySummary()
{
List <string> collNames = MongoTools.collectionNames(db);
string firstletter = "";
string vname = "";
int stationcode = 0;
string source = "";
int freq = 0;
int count = 0;
foreach (string collection in collNames)
{
//all station record collections start with an s_
string[] parts = collection.Split('_');
firstletter = parts[0];
if (firstletter == "s")
{
stationcode = Convert.ToInt32(parts[1]);
vname = parts[4];
if (vname == "PA")
{
continue;
}
source = parts[2];
freq = Convert.ToInt32(parts[5]);
if (source.Contains("IDEAM"))
{
source = "IDEAM";
}
else
{
source = "NOAA";
}
VariableMeta meta = AnnualSummary.getVariableMetaFromDB(vname, source, db);
VariableMonthly vm = new VariableMonthly(vname, freq);
addStation(stationcode);
addMonthlyRecord(stationcode, vm);
await sortByDate(stationcode, collection, meta);
count++;
}
}
insertManyMonthlySummary();
}
public void convert10min(string key)
{
List <string> collNames = MongoTools.collectionNames(db);
string vname = "";
int stationcode = 0;
string source = "";
int freq = 0;
int tenmincollections = 0;
foreach (string collection in collNames)
{
//all station record collections start with an s_
if (collection[0] == 's' && collection.Contains(key))
{
convertSingleCollection(collection);
}
}
}
public static bool checkIndexes()
{
IMongoDatabase db = MongoTools.connect("mongodb://localhost", "climaColombia");
//clean up bog buc with PA variable
List <string> collNames = MongoTools.collectionNames(db);
int withindex = 0;
int cleanCollection = 0;
foreach (string collection in collNames)
{
if (collection.Contains("Clean"))
{
cleanCollection++;
var coll = db.GetCollection <BsonDocument>(collection);
IMongoIndexManager <BsonDocument> index = coll.Indexes;
using (IAsyncCursor <BsonDocument> cursor = coll.Indexes.List())
{
while (cursor.MoveNext())
{
IEnumerable <BsonDocument> batch = cursor.Current;
foreach (BsonDocument b in batch)
{
if (b["key"].AsBsonDocument.Contains("time"))
{
withindex++;
}
}
}
}
}
}
bool success = false;
if (withindex == cleanCollection)
{
success = true;
}
return(success);
}
public static void removeCollections()
{
IMongoDatabase db = MongoTools.connect("mongodb://localhost", "climaColombia");
//clean up bog buc with PA variable
List <string> collNames = MongoTools.collectionNames(db);
foreach (string collection in collNames)
{
//all station record collections start with an s_
string[] parts = collection.Split('_');
string firstletter = parts[0];
if (firstletter == "s")
{
string vname = parts[4];
if (vname == "PA")
{
var coll = db.GetCollection <BsonDocument>(collection);
db.DropCollection(collection);
}
}
}
}
public async Task textSummaryStations(string dbname)
{
//summary based on total variables
List <string> collNames = MongoTools.collectionNames(db);
string firstletter = "";
string vname = "";
int stationcode = 0;
string source = "";
int freq = 0;
foreach (string collection in collNames)
{
//all station record collections start with an s_
string[] parts = collection.Split('_');
firstletter = parts[0];
if (firstletter == "s")
{
stationcode = Convert.ToInt32(parts[1]);
vname = parts[4];
if (vname == "PA")
{
continue;
}
source = parts[2];
freq = Convert.ToInt32(parts[5]);
VariableMeta meta = getVariableMetaFromDB(vname, source, db);
RecordMeta rm = new RecordMeta(vname, freq);
addStation(stationcode);
addRecord(stationcode, rm);
await getTotalRecords(collection, stationcode, vname);
await getDateLimits(collection, stationcode, vname);
await insideRange(collection, stationcode, meta);
}
}
annualStats();
insertManyRecordStationSummary();
printToSummary();
}
private List <string> getStationsColNames(StationGroup cityGroup)
{
List <string> collections = MongoTools.collectionNames(db);
List <string> stationCollections = new List <string>();
int scode = 0;
foreach (string col in collections)
{
if (col[0] == 's')
{
string[] parts = col.Split('_');
scode = Convert.ToInt32(parts[1]);
foreach (int code in cityGroup.stationcodes)
{
if (scode == code)
{
stationCollections.Add(col);
}
}
}
}
return(stationCollections);
}
private void checkTenMinAverages(object sender, EventArgs e)
{
MongoTools.checkAveraging();
}
public CleanRecords()
{
db = MongoTools.connect("mongodb://localhost/?maxPoolSize=1000", "climaColombia");
}
private void cloudClean(object sender, EventArgs e)
{
MongoTools.cloudCleaner();
}
private void checkIndexes(object sender, EventArgs e)
{
MongoTools.checkIndexes();
}