public static void UnlinkGNIS()
{
var osm = OpenFile(@"C:\Users\Alex\Downloads\Maine\maine-latest-internal.osm.pbf").ToArray();
var badsC = osm.Where(e => e.Tags != null && e.Tags.Contains("gnis:reviewed", "no") && e.UserName == "blackboxlogic+bot").ToDictionary(b => b.Id);
var bads = osm.Where(e => e.Tags != null && e.Tags.Contains("gnis:reviewed", "no") && e.UserName == "blackboxlogic+bot" && e is Node).ToDictionary(b => b.Id);
var prevs = OsmApiClient.GetElements(bads.Values.ToDictionary(b => new OsmGeoKey(b), b => b.Version - 1)).Result;
var news = new List <OsmGeo>();
var changes = new List <OsmGeo>();
foreach (var prev in prevs.Where(p => !p.Tags.ContainsKey("addr:street") &&
p.Id != 367794548 && p.Id != 367795033 && p.Id != 367794682 && p.Id != 367794721 && p.Id != 367795011 &&
p.Id != 367795009 && p.Id != 367794614 && p.Id != 367794612))
{
prev.Version++;
changes.Add(prev);
var bad = bads[prev.Id];
bad.Tags = new TagsCollection(bad.Tags.Where(t => t.Key.StartsWith("addr")));
bad.Id = -bad.Id;
news.Add(bad);
}
var change = Changes.FromGeos(news, changes, null);
FileSerializer.WriteXml("FIX unmerge gnis\\change.osc", change);
var changeTags = GetCommitTags("Separate addreses which should not have been merged into GNIS elements with reviewed=no");
var ids = Subjects.UploadChange(change, changeTags, "FIX unmerge gnis").Result;
//Console.WriteLine(string.Join(Environment.NewLine, bads.Values.Select(b => b.Type + " " + b.Id)));
}
public static void FixIslandPlaces()
{
var osm = OpenFile(@"C:\Users\Alex\Downloads\Maine\maine-latest-internal.osm.pbf").ToArray();
var index = OsmSharp.Db.Impl.Extensions.CreateSnapshotDb(new MemorySnapshotDb(osm));
var roadNames = new HashSet <string>(osm.Where(e => e.Tags != null && e.Tags.ContainsKey("highway") && e.Tags.ContainsKey("name")).Select(e => Connonical(e.Tags["name"])).Distinct());
var places = osm.Where(e => e.Tags != null && (e.Tags.ContainsKey("place") || e.Tags.ContainsKey("waterway")));
var islands = places.Where(e => e.Tags["place"] == "island" || e.Tags["place"] == "islet" || e.Tags.Any(t => t.Key == "name" && t.Value.Contains("island", StringComparison.OrdinalIgnoreCase))).ToArray();
var islandsByName = islands.Where(e => e.Tags.ContainsKey("name")).GroupBy(e => Connonical(e.Tags["name"])).ToDictionary();
var orphans = osm.Where(e => e.Tags != null && e.Tags.ContainsKey("addr:street") && !roadNames.Contains(Connonical(e.Tags["addr:street"])) &&
islandsByName.ContainsKey(Connonical(e.Tags["addr:street"]))).ToArray();
var matchedOrphans = orphans.Where(e => islandsByName[Connonical(e.Tags["addr:street"])].Any(i => Geometry.MinDistanceMeters(e.AsComplete(index).AsPosition(), i.AsComplete(index)) < 1000)).ToArray();
FileSerializer.WriteXml(@"Fix IslandPlaces\islands.osm", islands.WithChildren(index).AsOsm());
FileSerializer.WriteXml(@"Fix IslandPlaces\orphans.osm", orphans.WithChildren(index).AsOsm());
FileSerializer.WriteXml(@"Fix IslandPlaces\matchedOrphans.osm", matchedOrphans.WithChildren(index).AsOsm());
FileSerializer.WriteXml(@"Fix IslandPlaces\unmatchedOrphans.osm", orphans.Except(matchedOrphans).WithChildren(index).AsOsm());
foreach (var orphan in orphans)
{
orphan.Tags["addr:place"] = orphan.Tags["addr:street"];
orphan.Tags.RemoveKey("addr:street");
}
var change = Changes.FromGeos(null, orphans, null);
var changeTags = GetCommitTags("Moving addr:street tag to addr:place for island addresses.");
var ids = Subjects.UploadChange(change, changeTags, "Fix IslandPlaces").Result;
}
public static void FixPlaces()
{
var osm = OpenFile(@"C:\Users\Alex\Downloads\maine-latest-internal.osm.pbf").ToArray();
var index = OsmSharp.Db.Impl.Extensions.CreateSnapshotDb(new MemorySnapshotDb(osm));
var roadsByName = osm.Where(e => e.Tags != null && e.Tags.ContainsKey("highway") && e.Tags.ContainsKey("name"))
.GroupBy(e => Connonical(e.Tags["name"])).ToDictionary();
var placesByName = osm.Where(e => e.Tags != null && e.Tags.ContainsKey("name") && (e.Tags.ContainsKey("place") || e.Tags.ContainsKey("waterway")))
.GroupBy(e => Connonical(e.Tags["name"])).ToDictionary();
var orphans = osm.Where(e => e.Tags != null && e.Tags.Contains("addr:state", "ME") && e.Tags.ContainsKey("addr:street") && !roadsByName.ContainsKey(Connonical(e.Tags["addr:street"])) &&
placesByName.ContainsKey(Connonical(e.Tags["addr:street"])) &&
placesByName[Connonical(e.Tags["addr:street"])].Any(
p => Geometry.MinDistanceMeters(e.AsComplete(index).AsPosition(), p.AsComplete(index)) < 1000)).ToArray();
var sad = osm.Where(e => e.Tags != null && e.Tags.ContainsKey("addr:street") && e.Tags["addr:street"].Contains("Island") && !roadsByName.ContainsKey(Connonical(e.Tags["addr:street"])) &&
(!placesByName.ContainsKey(Connonical(e.Tags["addr:street"])) ||
!placesByName[Connonical(e.Tags["addr:street"])].Any(
p => Geometry.MinDistanceMeters(e.AsComplete(index).AsPosition(), p.AsComplete(index)) < 1000))).ToArray();
FileSerializer.WriteXml(@"Fix IslandPlaces\islands.osm", placesByName.Values.SelectMany(ps => ps).WithChildren(index).AsOsm());
FileSerializer.WriteXml(@"Fix IslandPlaces\orphans.osm", orphans.WithChildren(index).AsOsm());
FileSerializer.WriteXml(@"Fix IslandPlaces\sadOrphans.osm", sad.WithChildren(index).AsOsm());
foreach (var orphan in orphans)
{
orphan.Tags["addr:place"] = orphan.Tags["addr:street"];
orphan.Tags.RemoveKey("addr:street");
}
var change = Changes.FromGeos(null, orphans, null);
var changeTags = GetCommitTags("Moving addr:street tag to addr:place for island addresses.");
var ids = Subjects.UploadChange(change, changeTags, "Fix IslandPlaces").Result;
}
public static void FixPhones(ILoggerFactory loggerFactory, IConfigurationRoot config, OsmGeo scope)
{
var rejectLogger = loggerFactory.CreateLogger("Unfixable");
Action <OsmGeo> rejector = a => rejectLogger.LogWarning($"Rejected {a.Type}:{a.Id} {a.Tags[Tag]}");
var osmApi = new OsmApiEnder(loggerFactory.CreateLogger(typeof(OsmApiEnder)),
config["OsmApiUrl"], config["OsmUsername"], config["OsmPassword"], ChangeTags);
var badPhones = OverpassApi.Get(BadPhoneQuery(scope, Tag)).GetElements();
var betterPhones = FixPhones(badPhones, Tag, rejector).ToArray();
var change = Changes.FromGeos(null, betterPhones, null, nameof(OsmPipeline));
osmApi.Upload(change);
}
public static OsmChange UndoTagFromElements(Tag tagToRemove, OsmGeo[] osmGeos, Tag?replaceWith = null)
{
var stillBad = osmGeos.Where(e => e.Tags != null && e.Tags.Contains(tagToRemove) && e.Visible == true).ToArray();
foreach (var e in stillBad)
{
e.Tags.RemoveKeyValue(tagToRemove);
if (replaceWith != null)
{
e.Tags.Add(replaceWith.Value);
}
}
var theFix = Changes.FromGeos(null, stillBad, null);
return(theFix);
}
public static OsmChange Merge(Osm reference, Osm subject, string scopeName, GeoJsonAPISource.Municipality municipality)
{
Log = Log ?? Static.LogFactory.CreateLogger(typeof(Conflate));
// Can blacklist subject elements also! Fixme: colisions between different element types with the same ID
var subjectElements = subject.GetElements().Where(e => !municipality.BlackList.Contains(e.Id.Value));
var subjectElementIndex = new ElementIndex(subjectElements.ToArray());
List <OsmGeo> create = new List <OsmGeo>(reference.Nodes);
List <OsmGeo> modify = new List <OsmGeo>();
List <OsmGeo> delete = new List <OsmGeo>();
Log.LogInformation("Starting conflation, matching by tags");
MergeNodesByTags(new[] { "addr:street", "addr:housenumber", "addr:unit" }, subjectElementIndex, municipality.WhiteList, create, modify);
MergeNodesByTags(new[] { "addr:street", "addr:housenumber" }, subjectElementIndex, municipality.WhiteList, create, modify);
MergeNodesByTags(new[] { "name" }, subjectElementIndex, municipality.WhiteList, create, modify);
Log.LogInformation("Starting conflation, matching by geometry");
MergeNodesByGeometry(subjectElementIndex, municipality.WhiteList, create, modify);
ValidateNamesMatch(subjectElementIndex, create.Concat(modify), "highway", "addr:street",
(element, key) => ShouldStreetBeAPlace(element, subjectElementIndex));
ValidateNamesMatch(subjectElementIndex, create.Concat(modify), "place", "addr:place");
Log.LogInformation($"Writing {scopeName} review files");
var review = GatherExceptions(municipality.WhiteList, municipality.IgnoreList, subjectElementIndex, create, delete, modify);
WriteToFileWithChildrenIfAny(scopeName + "/Conflated.Review.osm", review, subjectElementIndex);
WriteToFileWithChildrenIfAny(scopeName + "/Conflated.Create.osm", create, subjectElementIndex);
WriteToFileWithChildrenIfAny(scopeName + "/Conflated.Delete.osm", delete, subjectElementIndex);
WriteToFileWithChildrenIfAny(scopeName + "/Conflated.Modify.osm", modify, subjectElementIndex);
RemoveReviewTags(create, delete, modify);
var change = Changes.FromGeos(create, modify, delete, "OsmPipeline");
LogSummary(change, review);
return(change);
}
public static void FixAddrStreetFormat()
{
// or @"C:\Users\Alex\Downloads\Maine\maine-latest.osm\maine-latest.osm"
var osm = OpenFile(@"C:\Users\Alex\Downloads\Maine\maine-latest-internal.osm.pbf");
var index = osm.ToDictionary(e => new OsmGeoKey(e));
var stringdex = index.Values.ToDictionary(e => e.Type.ToString() + e.Id);
var canonNamesToCompleteWays = index.Values
.OfType <Way>()
.Where(e => e.Tags != null && e.Tags.ContainsKey("highway") && e.Tags.ContainsKey("name"))
.Select(w => w.AsComplete(stringdex))
.GroupBy(e => Connonical(e.Tags["name"]))
.ToDictionary();
var addrStreets = index.Values.Where(e => e.Tags != null && e.Tags.Contains("addr:state", "ME") && e.Tags.ContainsKey("addr:street"))
.Select(element => new { element, addrStreet = element.Tags["addr:street"], canon = Connonical(element.Tags["addr:street"]), position = element.AsComplete(stringdex).AsPosition() })
.Where(e => canonNamesToCompleteWays.ContainsKey(e.canon))
.ToArray();
var changes = new List <OsmGeo>();
foreach (var addr in addrStreets)
{
var roadMatches = canonNamesToCompleteWays[addr.canon]
.Where(r => Geometry.MinDistanceMeters(addr.position, r) < 1000).ToArray();
var roadNameMatches = roadMatches.Select(r => r.Tags["name"]).Distinct().ToArray();
if (roadNameMatches.Length == 0)
{
}
else if (roadNameMatches.Length > 1)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("conflict " + addr.element.Type + " " + addr.element.Id + " -> " + string.Join(", ", roadNameMatches));
Console.ForegroundColor = ConsoleColor.White;
roadMatches.First().Tags.Add("fixme",
"Different road segments have name formats conflict. Check the street sign and fix: " +
string.Join(" vs ", roadNameMatches.Select(n => '[' + n + ']')));
//changes.Add(roadMatches.First().AsSimple());
}
else if (roadNameMatches[0] != addr.addrStreet)
{
//if (WhitespacesDiffers(addr.addrStreet, roadNameMatches[0]) && addr.element.UserName.StartsWith("blackboxlogic"))
//{
// var did = false;
// foreach (var road in roadMatches)
// {
// if (!road.Tags.Contains("official_name", addr.addrStreet))
// {
// did = true;
// road.Tags.AddOrReplace("official_name", addr.addrStreet);
// changes.Add(road.AsOsmGeo());
// }
// }
// if (did) Console.WriteLine("O\t" + roadMatches.Length + "x " + roadNameMatches.First() + " -> " + addr.addrStreet);
//}
Console.WriteLine(addr.addrStreet + " -> " + roadNameMatches.First());
addr.element.Tags["addr:street"] = roadNameMatches.First();
changes.Add(addr.element);
}
}
var change = Changes.FromGeos(null, changes, null);
var ids = Subjects.UploadChange(change, GetCommitTags("Adjusting addr:street format to match nearby street name (case, punctuation, space)"), "Fix addrStreet Format").Result;
//var streetNameConflicts = connonicalStreetNames.Where(c => c.Value.Length > 1)
// .ToDictionary(c => c.Key, c => FightingWays(c.Value.SelectMany(n => streetNamesToWays[n].Ways)));
//Console.WriteLine(string.Join(Environment.NewLine,
// streetNameConflicts.Where(kvp => kvp.Value.Any()).Select(kvp => kvp.Key + ": " + "\n\t" +
// string.Join("\n\t", kvp.Value.Select(w => "osm.org/way/" + w.Id + "\t" + wayIndex[w.Id].Tags["name"])))));
}