static async Task insertAll(IMongoCollection <BsonDocument> rawData, IMongoCollection <BsonDocument> cleanData)
{
GlobalMapTiles.GlobalMercator proj = new GlobalMapTiles.GlobalMercator();
BsonDocument all = new BsonDocument();
long count = 0;
using (var cursor = await rawData.FindAsync(all)) {
while (await cursor.MoveNextAsync())
{
var batch = cursor.Current;
foreach (var doc in batch)
{
BsonArray a = doc["map"]["nearbyProperties"].AsBsonArray;
BsonDocument[] docs = new BsonDocument[a.Count];
for (int i = 0; i < a.Count; i++)
{
var filter = Builders <BsonDocument> .Filter.Eq("_id", a[i][0]);
//Task<long> previousCount = cleanData.Find(filter).CountAsync();
//previousCount.Wait();
//if(previousCount.Result>0) {continue;}
if (!(a[i][7][0].BsonType == BsonType.Int32))
{
continue;
}
double lat = (double)a[i][1].AsInt32 / 1000000.0;
double lng = (double)a[i][2].AsInt32 / 1000000.0;
int price = (int)a[i][7][0].AsInt32;
string quad = proj.latlngToQuadKey(lat, lng, 23);
var update = Builders <BsonDocument> .Update
.Set("lat", lat)
.Set("lng", lng)
.Set("price", price)
.Set("bed", a[i][7][1])
.Set("bath", a[i][7][2])
.Set("sf", a[i][7][3])
.Set("type", a[i][7][4])
.Set("img", a[i][7][5])
.Set("quad", quad);
var options = new UpdateOptions {
IsUpsert = true
};
var result = await cleanData.UpdateOneAsync(filter, update, options);
count++;
}
}
Console.WriteLine(count);
}
}
}
static async Task insertDFW_quadKey(IMongoCollection <BsonDocument> rawData, IMongoCollection <BsonDocument> cleanData)
{
BsonDocument all = new BsonDocument();
GlobalMapTiles.GlobalMercator proj = new GlobalMapTiles.GlobalMercator();
int minZoom = 9;
string quadDallas = proj.latlngToQuadKey(32.7805664, -96.8081442, minZoom);
string quadFtWorth = proj.latlngToQuadKey(32.7586487, -97.3324023, minZoom);
List <string> quads = new List <string>();
quads.Add(quadDallas);
quads.Add(quadFtWorth);
int[][] tilesXYZ = new int[quads.Count][];
double[][] bounds = new double[quads.Count][];
for (int i = 0; i < quads.Count; i++)
{
tilesXYZ[i] = proj.quadKeyToTileXY(quads[i]);
bounds[i] = proj.tileLatLngBounds(tilesXYZ[i][0], tilesXYZ[i][1], tilesXYZ[i][2]);
}
//filter = builder.Gt("lat", bounds[0]);
//filter = filter & builder.Lt("lat", bounds[1]);
//filter = filter & builder.Gt("lng", bounds[2]);
//filter = filter & builder.Lt("lng", bounds[3]);
long count = 0;
using (var cursor = await rawData.FindAsync(all)) {
while (await cursor.MoveNextAsync())
{
var batch = cursor.Current;
foreach (var doc in batch)
{
BsonArray a = doc["map"]["nearbyProperties"].AsBsonArray;
BsonDocument[] docs = new BsonDocument[a.Count];
for (int i = 0; i < a.Count; i++)
{
var filterID = Builders <BsonDocument> .Filter.Eq("_id", a[i][0]);
//Task<long> previousCount = cleanData.Find(filter).CountAsync();
//previousCount.Wait();
//if(previousCount.Result>0) {continue;}
if (!(a[i][7][0].BsonType == BsonType.Int32))
{
continue;
}
double lat = (double)a[i][1].AsInt32 / 1000000.0;
double lng = (double)a[i][2].AsInt32 / 1000000.0;
bool contained = false;
for (int j = 0; j < bounds.Length; j++)
{
if (bounds[i][0] < lat && lat < bounds[i][1] && bounds[i][2] < lng && lng < bounds[i][3])
{
contained = true;
}
}
if (contained)
{
int price = (int)a[i][7][0].AsInt32;
string quad = proj.latlngToQuadKey(lat, lng, 23);
//MongoDB.Driver.GeoJsonObjectModel.GeoJson2DCoordinates pt = new MongoDB.Driver.GeoJsonObjectModel.GeoJson2DCoordinates(lng, lat);
//BsonDocument ptBsonDocument = MongoDB.Driver.GeoJsonObjectModel.GeoJson.Point(pt).ToBsonDocument();
//BsonDocument geoJson = new BsonDocument()
// {
// { "type", "Point"},
// { "coordinates", ptBsonDocument }
// };
var update = Builders <BsonDocument> .Update
//.Set("loc", geoJson)
.Set("lat", lat)
.Set("lng", lng)
.Set("price", price)
.Set("bed", a[i][7][1])
.Set("bath", a[i][7][2])
.Set("sf", a[i][7][3])
.Set("type", a[i][7][4])
.Set("img", a[i][7][5])
.Set("quad", quad)
;
var options = new UpdateOptions {
IsUpsert = true
};
var result = await cleanData.UpdateOneAsync(filterID, update, options);
count++;
}
}
}
Console.WriteLine(count);
}
}
}
