/// <summary>
/// Loads up TSV data into RAM for use.
/// </summary>
/// <param name="filename">the geomData file to parse. Matching .tagsData file is assumed.</param>
/// <returns>a list of storedOSMelements</returns>
public static List <DbTables.Place> ReadStoredElementsFileToMemory(string filename)
{
StreamReader srGeo = new StreamReader(filename);
StreamReader srTags = new StreamReader(filename.Replace(".geomData", ".tagsData"));
List <DbTables.Place> lm = new List <DbTables.Place>(8000);
List <PlaceTags> tagsTemp = new List <PlaceTags>(8000);
ILookup <long, PlaceTags> tagDict;
while (!srTags.EndOfStream)
{
string line = srTags.ReadLine();
PlaceTags tag = ConvertSingleTsvTag(line);
tagsTemp.Add(tag);
}
srTags.Close(); srTags.Dispose();
tagDict = tagsTemp.ToLookup(k => k.SourceItemId, v => v);
while (!srGeo.EndOfStream)
{
string line = srGeo.ReadLine();
var sw = ConvertSingleTsvPlace(line);
sw.Tags = tagDict[sw.SourceItemID].ToList();
lm.Add(sw);
}
srGeo.Close(); srGeo.Dispose();
if (lm.Count() == 0)
{
Log.WriteLog("No entries for " + filename + "? why?");
}
Log.WriteLog("EOF Reached for " + filename + " at " + DateTime.Now);
return(lm);
}
public ActionResult DeleteConfirmed(int id)
{
PlaceTags placeTags = db.PlaceTags.Find(id);
db.PlaceTags.Remove(placeTags);
db.SaveChanges();
return(RedirectToAction("Index"));
}
public static PlaceTags ConvertSingleTsvTag(string sw)
{
var source = sw.AsSpan();
PlaceTags entry = new PlaceTags();
entry.SourceItemId = source.SplitNext('\t').ToLong();
entry.SourceItemType = source.SplitNext('\t').ToInt();
entry.Key = source.SplitNext('\t').ToString();
entry.Value = source.ToString();
return(entry);
}
public ActionResult Edit([Bind(Include = "PlaceTagID,PlaceID,TagID")] PlaceTags placeTags)
{
if (ModelState.IsValid)
{
db.Entry(placeTags).State = EntityState.Modified;
db.SaveChanges();
return(RedirectToAction("Index"));
}
ViewBag.TagID = new SelectList(db.Tag, "tagID", "name", placeTags.TagID);
ViewBag.PlaceID = new SelectList(db.Place, "PlaceID", "Name", placeTags.PlaceID);
return(View(placeTags));
}
// GET: PlaceTags/Details/5
public ActionResult Details(int?id)
{
if (id == null)
{
return(new HttpStatusCodeResult(HttpStatusCode.BadRequest));
}
PlaceTags placeTags = db.PlaceTags.Find(id);
if (placeTags == null)
{
return(HttpNotFound());
}
return(View(placeTags));
}
// GET: PlaceTags/Edit/5
public ActionResult Edit(int?id)
{
if (id == null)
{
return(new HttpStatusCodeResult(HttpStatusCode.BadRequest));
}
PlaceTags placeTags = db.PlaceTags.Find(id);
if (placeTags == null)
{
return(HttpNotFound());
}
ViewBag.TagID = new SelectList(db.Tag, "tagID", "name", placeTags.TagID);
ViewBag.PlaceID = new SelectList(db.Place, "PlaceID", "Name", placeTags.PlaceID);
return(View(placeTags));
}
private static void loadProcessedData()
{
Log.WriteLog("Starting load from processed files at " + DateTime.Now);
System.Diagnostics.Stopwatch fullProcess = new System.Diagnostics.Stopwatch();
fullProcess.Start();
PraxisContext db = new PraxisContext();
db.Database.SetCommandTimeout(Int32.MaxValue);
db.ChangeTracker.AutoDetectChangesEnabled = false;
List <string> geomFilenames = Directory.EnumerateFiles(config["OutputDataFolder"], "*.geomData").ToList();
List <string> tagsFilenames = Directory.EnumerateFiles(config["OutputDataFolder"], "*.tagsData").ToList();
if (config["KeepElementsInMemory"] == "True") //ignore DB, doing some one-off operation.
{
//Skip database work. Use an in-memory list for a temporary operation.
foreach (var fileName in geomFilenames)
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
Log.WriteLog("Loading " + fileName + " to memory at " + DateTime.Now);
var entries = File.ReadAllLines(fileName);
foreach (var entry in entries)
{
DbTables.Place stored = GeometrySupport.ConvertSingleTsvPlace(entry);
memorySource.Add(stored);
}
Log.WriteLog("File loaded to memory in " + sw.Elapsed);
sw.Stop();
}
foreach (var fileName in tagsFilenames)
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
Log.WriteLog("Loading " + fileName + " to memory at " + DateTime.Now);
var entries = File.ReadAllLines(fileName);
foreach (var entry in entries)
{
PlaceTags stored = GeometrySupport.ConvertSingleTsvTag(entry);
var taggedGeo = memorySource.First(m => m.SourceItemType == stored.SourceItemType && m.SourceItemID == stored.SourceItemId);
//MemorySource will need to be a more efficient collection for searching if this is to be a major feature, but this functions.
taggedGeo.Tags.Add(stored);
}
Log.WriteLog("File applied to memory in " + sw.Elapsed);
sw.Stop();
}
return;
}
else if (config["UseMariaDBInFile"] == "True") //Use the LOAD DATA INFILE command to skip the EF for loading.
{
foreach (var fileName in geomFilenames)
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
var mariaPath = fileName.Replace("\\", "\\\\");
db.Database.ExecuteSqlRaw("LOAD DATA INFILE '" + mariaPath + "' IGNORE INTO TABLE Places fields terminated by '\t' lines terminated by '\r\n' (sourceItemID, sourceItemType, @elementGeometry, AreaSize, privacyId) SET elementGeometry = ST_GeomFromText(@elementGeometry) ");
sw.Stop();
Log.WriteLog("Geometry loaded from " + fileName + " in " + sw.Elapsed);
File.Move(fileName, fileName + "done");
}
foreach (var fileName in tagsFilenames)
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
var mariaPath = fileName.Replace("\\", "\\\\");
db.Database.ExecuteSqlRaw("LOAD DATA INFILE '" + mariaPath + "' IGNORE INTO TABLE PlaceTags fields terminated by '\t' lines terminated by '\r\n' (SourceItemId, SourceItemType, `key`, `value`)");
sw.Stop();
Log.WriteLog("Tags loaded from " + fileName + " in " + sw.Elapsed);
File.Move(fileName, fileName + "done");
}
}
else //Main path.
{
Parallel.ForEach(geomFilenames, fileName =>
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
var db = new PraxisContext();
db.Database.SetCommandTimeout(Int32.MaxValue);
db.ChangeTracker.AutoDetectChangesEnabled = false;
var lines = File.ReadAllLines(fileName); //Might be faster to use streams and dodge the memory allocation?
foreach (var line in lines)
{
db.Places.Add(GeometrySupport.ConvertSingleTsvPlace(line));
}
db.SaveChanges();
sw.Stop();
Log.WriteLog("Geometry loaded from " + fileName + " in " + sw.Elapsed);
File.Move(fileName, fileName + "done");
});
Parallel.ForEach(tagsFilenames, fileName =>
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
var db = new PraxisContext();
db.Database.SetCommandTimeout(Int32.MaxValue);
db.ChangeTracker.AutoDetectChangesEnabled = false;
var lines = File.ReadAllLines(fileName);
foreach (var line in lines)
{
db.PlaceTags.Add(GeometrySupport.ConvertSingleTsvTag(line));
}
db.SaveChanges();
sw.Stop();
Log.WriteLog("Tags loaded from " + fileName + " in " + sw.Elapsed);
File.Move(fileName, fileName + "done");
});
}
fullProcess.Stop();
Log.WriteLog("Files processed in " + fullProcess.Elapsed);
fullProcess.Restart();
db.RecreateIndexes();
fullProcess.Stop();
Log.WriteLog("Indexes generated in " + fullProcess.Elapsed);
}
