public static void UnlinkGNIS() { var osm = OpenFile(@"C:\Users\Alex\Downloads\Maine\maine-latest-internal.osm.pbf").ToArray(); var badsC = osm.Where(e => e.Tags != null && e.Tags.Contains("gnis:reviewed", "no") && e.UserName == "blackboxlogic+bot").ToDictionary(b => b.Id); var bads = osm.Where(e => e.Tags != null && e.Tags.Contains("gnis:reviewed", "no") && e.UserName == "blackboxlogic+bot" && e is Node).ToDictionary(b => b.Id); var prevs = OsmApiClient.GetElements(bads.Values.ToDictionary(b => new OsmGeoKey(b), b => b.Version - 1)).Result; var news = new List <OsmGeo>(); var changes = new List <OsmGeo>(); foreach (var prev in prevs.Where(p => !p.Tags.ContainsKey("addr:street") && p.Id != 367794548 && p.Id != 367795033 && p.Id != 367794682 && p.Id != 367794721 && p.Id != 367795011 && p.Id != 367795009 && p.Id != 367794614 && p.Id != 367794612)) { prev.Version++; changes.Add(prev); var bad = bads[prev.Id]; bad.Tags = new TagsCollection(bad.Tags.Where(t => t.Key.StartsWith("addr"))); bad.Id = -bad.Id; news.Add(bad); } var change = Changes.FromGeos(news, changes, null); FileSerializer.WriteXml("FIX unmerge gnis\\change.osc", change); var changeTags = GetCommitTags("Separate addreses which should not have been merged into GNIS elements with reviewed=no"); var ids = Subjects.UploadChange(change, changeTags, "FIX unmerge gnis").Result; //Console.WriteLine(string.Join(Environment.NewLine, bads.Values.Select(b => b.Type + " " + b.Id))); }
public static void FixIslandPlaces() { var osm = OpenFile(@"C:\Users\Alex\Downloads\Maine\maine-latest-internal.osm.pbf").ToArray(); var index = OsmSharp.Db.Impl.Extensions.CreateSnapshotDb(new MemorySnapshotDb(osm)); var roadNames = new HashSet <string>(osm.Where(e => e.Tags != null && e.Tags.ContainsKey("highway") && e.Tags.ContainsKey("name")).Select(e => Connonical(e.Tags["name"])).Distinct()); var places = osm.Where(e => e.Tags != null && (e.Tags.ContainsKey("place") || e.Tags.ContainsKey("waterway"))); var islands = places.Where(e => e.Tags["place"] == "island" || e.Tags["place"] == "islet" || e.Tags.Any(t => t.Key == "name" && t.Value.Contains("island", StringComparison.OrdinalIgnoreCase))).ToArray(); var islandsByName = islands.Where(e => e.Tags.ContainsKey("name")).GroupBy(e => Connonical(e.Tags["name"])).ToDictionary(); var orphans = osm.Where(e => e.Tags != null && e.Tags.ContainsKey("addr:street") && !roadNames.Contains(Connonical(e.Tags["addr:street"])) && islandsByName.ContainsKey(Connonical(e.Tags["addr:street"]))).ToArray(); var matchedOrphans = orphans.Where(e => islandsByName[Connonical(e.Tags["addr:street"])].Any(i => Geometry.MinDistanceMeters(e.AsComplete(index).AsPosition(), i.AsComplete(index)) < 1000)).ToArray(); FileSerializer.WriteXml(@"Fix IslandPlaces\islands.osm", islands.WithChildren(index).AsOsm()); FileSerializer.WriteXml(@"Fix IslandPlaces\orphans.osm", orphans.WithChildren(index).AsOsm()); FileSerializer.WriteXml(@"Fix IslandPlaces\matchedOrphans.osm", matchedOrphans.WithChildren(index).AsOsm()); FileSerializer.WriteXml(@"Fix IslandPlaces\unmatchedOrphans.osm", orphans.Except(matchedOrphans).WithChildren(index).AsOsm()); foreach (var orphan in orphans) { orphan.Tags["addr:place"] = orphan.Tags["addr:street"]; orphan.Tags.RemoveKey("addr:street"); } var change = Changes.FromGeos(null, orphans, null); var changeTags = GetCommitTags("Moving addr:street tag to addr:place for island addresses."); var ids = Subjects.UploadChange(change, changeTags, "Fix IslandPlaces").Result; }
public static void FixPlaces() { var osm = OpenFile(@"C:\Users\Alex\Downloads\maine-latest-internal.osm.pbf").ToArray(); var index = OsmSharp.Db.Impl.Extensions.CreateSnapshotDb(new MemorySnapshotDb(osm)); var roadsByName = osm.Where(e => e.Tags != null && e.Tags.ContainsKey("highway") && e.Tags.ContainsKey("name")) .GroupBy(e => Connonical(e.Tags["name"])).ToDictionary(); var placesByName = osm.Where(e => e.Tags != null && e.Tags.ContainsKey("name") && (e.Tags.ContainsKey("place") || e.Tags.ContainsKey("waterway"))) .GroupBy(e => Connonical(e.Tags["name"])).ToDictionary(); var orphans = osm.Where(e => e.Tags != null && e.Tags.Contains("addr:state", "ME") && e.Tags.ContainsKey("addr:street") && !roadsByName.ContainsKey(Connonical(e.Tags["addr:street"])) && placesByName.ContainsKey(Connonical(e.Tags["addr:street"])) && placesByName[Connonical(e.Tags["addr:street"])].Any( p => Geometry.MinDistanceMeters(e.AsComplete(index).AsPosition(), p.AsComplete(index)) < 1000)).ToArray(); var sad = osm.Where(e => e.Tags != null && e.Tags.ContainsKey("addr:street") && e.Tags["addr:street"].Contains("Island") && !roadsByName.ContainsKey(Connonical(e.Tags["addr:street"])) && (!placesByName.ContainsKey(Connonical(e.Tags["addr:street"])) || !placesByName[Connonical(e.Tags["addr:street"])].Any( p => Geometry.MinDistanceMeters(e.AsComplete(index).AsPosition(), p.AsComplete(index)) < 1000))).ToArray(); FileSerializer.WriteXml(@"Fix IslandPlaces\islands.osm", placesByName.Values.SelectMany(ps => ps).WithChildren(index).AsOsm()); FileSerializer.WriteXml(@"Fix IslandPlaces\orphans.osm", orphans.WithChildren(index).AsOsm()); FileSerializer.WriteXml(@"Fix IslandPlaces\sadOrphans.osm", sad.WithChildren(index).AsOsm()); foreach (var orphan in orphans) { orphan.Tags["addr:place"] = orphan.Tags["addr:street"]; orphan.Tags.RemoveKey("addr:street"); } var change = Changes.FromGeos(null, orphans, null); var changeTags = GetCommitTags("Moving addr:street tag to addr:place for island addresses."); var ids = Subjects.UploadChange(change, changeTags, "Fix IslandPlaces").Result; }
public static void FixPhones(ILoggerFactory loggerFactory, IConfigurationRoot config, OsmGeo scope) { var rejectLogger = loggerFactory.CreateLogger("Unfixable"); Action <OsmGeo> rejector = a => rejectLogger.LogWarning($"Rejected {a.Type}:{a.Id} {a.Tags[Tag]}"); var osmApi = new OsmApiEnder(loggerFactory.CreateLogger(typeof(OsmApiEnder)), config["OsmApiUrl"], config["OsmUsername"], config["OsmPassword"], ChangeTags); var badPhones = OverpassApi.Get(BadPhoneQuery(scope, Tag)).GetElements(); var betterPhones = FixPhones(badPhones, Tag, rejector).ToArray(); var change = Changes.FromGeos(null, betterPhones, null, nameof(OsmPipeline)); osmApi.Upload(change); }
public static OsmChange UndoTagFromElements(Tag tagToRemove, OsmGeo[] osmGeos, Tag?replaceWith = null) { var stillBad = osmGeos.Where(e => e.Tags != null && e.Tags.Contains(tagToRemove) && e.Visible == true).ToArray(); foreach (var e in stillBad) { e.Tags.RemoveKeyValue(tagToRemove); if (replaceWith != null) { e.Tags.Add(replaceWith.Value); } } var theFix = Changes.FromGeos(null, stillBad, null); return(theFix); }
public static OsmChange Merge(Osm reference, Osm subject, string scopeName, GeoJsonAPISource.Municipality municipality) { Log = Log ?? Static.LogFactory.CreateLogger(typeof(Conflate)); // Can blacklist subject elements also! Fixme: colisions between different element types with the same ID var subjectElements = subject.GetElements().Where(e => !municipality.BlackList.Contains(e.Id.Value)); var subjectElementIndex = new ElementIndex(subjectElements.ToArray()); List <OsmGeo> create = new List <OsmGeo>(reference.Nodes); List <OsmGeo> modify = new List <OsmGeo>(); List <OsmGeo> delete = new List <OsmGeo>(); Log.LogInformation("Starting conflation, matching by tags"); MergeNodesByTags(new[] { "addr:street", "addr:housenumber", "addr:unit" }, subjectElementIndex, municipality.WhiteList, create, modify); MergeNodesByTags(new[] { "addr:street", "addr:housenumber" }, subjectElementIndex, municipality.WhiteList, create, modify); MergeNodesByTags(new[] { "name" }, subjectElementIndex, municipality.WhiteList, create, modify); Log.LogInformation("Starting conflation, matching by geometry"); MergeNodesByGeometry(subjectElementIndex, municipality.WhiteList, create, modify); ValidateNamesMatch(subjectElementIndex, create.Concat(modify), "highway", "addr:street", (element, key) => ShouldStreetBeAPlace(element, subjectElementIndex)); ValidateNamesMatch(subjectElementIndex, create.Concat(modify), "place", "addr:place"); Log.LogInformation($"Writing {scopeName} review files"); var review = GatherExceptions(municipality.WhiteList, municipality.IgnoreList, subjectElementIndex, create, delete, modify); WriteToFileWithChildrenIfAny(scopeName + "/Conflated.Review.osm", review, subjectElementIndex); WriteToFileWithChildrenIfAny(scopeName + "/Conflated.Create.osm", create, subjectElementIndex); WriteToFileWithChildrenIfAny(scopeName + "/Conflated.Delete.osm", delete, subjectElementIndex); WriteToFileWithChildrenIfAny(scopeName + "/Conflated.Modify.osm", modify, subjectElementIndex); RemoveReviewTags(create, delete, modify); var change = Changes.FromGeos(create, modify, delete, "OsmPipeline"); LogSummary(change, review); return(change); }
public static void FixAddrStreetFormat() { // or @"C:\Users\Alex\Downloads\Maine\maine-latest.osm\maine-latest.osm" var osm = OpenFile(@"C:\Users\Alex\Downloads\Maine\maine-latest-internal.osm.pbf"); var index = osm.ToDictionary(e => new OsmGeoKey(e)); var stringdex = index.Values.ToDictionary(e => e.Type.ToString() + e.Id); var canonNamesToCompleteWays = index.Values .OfType <Way>() .Where(e => e.Tags != null && e.Tags.ContainsKey("highway") && e.Tags.ContainsKey("name")) .Select(w => w.AsComplete(stringdex)) .GroupBy(e => Connonical(e.Tags["name"])) .ToDictionary(); var addrStreets = index.Values.Where(e => e.Tags != null && e.Tags.Contains("addr:state", "ME") && e.Tags.ContainsKey("addr:street")) .Select(element => new { element, addrStreet = element.Tags["addr:street"], canon = Connonical(element.Tags["addr:street"]), position = element.AsComplete(stringdex).AsPosition() }) .Where(e => canonNamesToCompleteWays.ContainsKey(e.canon)) .ToArray(); var changes = new List <OsmGeo>(); foreach (var addr in addrStreets) { var roadMatches = canonNamesToCompleteWays[addr.canon] .Where(r => Geometry.MinDistanceMeters(addr.position, r) < 1000).ToArray(); var roadNameMatches = roadMatches.Select(r => r.Tags["name"]).Distinct().ToArray(); if (roadNameMatches.Length == 0) { } else if (roadNameMatches.Length > 1) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("conflict " + addr.element.Type + " " + addr.element.Id + " -> " + string.Join(", ", roadNameMatches)); Console.ForegroundColor = ConsoleColor.White; roadMatches.First().Tags.Add("fixme", "Different road segments have name formats conflict. Check the street sign and fix: " + string.Join(" vs ", roadNameMatches.Select(n => '[' + n + ']'))); //changes.Add(roadMatches.First().AsSimple()); } else if (roadNameMatches[0] != addr.addrStreet) { //if (WhitespacesDiffers(addr.addrStreet, roadNameMatches[0]) && addr.element.UserName.StartsWith("blackboxlogic")) //{ // var did = false; // foreach (var road in roadMatches) // { // if (!road.Tags.Contains("official_name", addr.addrStreet)) // { // did = true; // road.Tags.AddOrReplace("official_name", addr.addrStreet); // changes.Add(road.AsOsmGeo()); // } // } // if (did) Console.WriteLine("O\t" + roadMatches.Length + "x " + roadNameMatches.First() + " -> " + addr.addrStreet); //} Console.WriteLine(addr.addrStreet + " -> " + roadNameMatches.First()); addr.element.Tags["addr:street"] = roadNameMatches.First(); changes.Add(addr.element); } } var change = Changes.FromGeos(null, changes, null); var ids = Subjects.UploadChange(change, GetCommitTags("Adjusting addr:street format to match nearby street name (case, punctuation, space)"), "Fix addrStreet Format").Result; //var streetNameConflicts = connonicalStreetNames.Where(c => c.Value.Length > 1) // .ToDictionary(c => c.Key, c => FightingWays(c.Value.SelectMany(n => streetNamesToWays[n].Ways))); //Console.WriteLine(string.Join(Environment.NewLine, // streetNameConflicts.Where(kvp => kvp.Value.Any()).Select(kvp => kvp.Key + ": " + "\n\t" + // string.Join("\n\t", kvp.Value.Select(w => "osm.org/way/" + w.Id + "\t" + wayIndex[w.Id].Tags["name"]))))); }