public static async Task <long[]> UploadChange(string changeFile, string description)
{
var change = FileSerializer.ReadXml <OsmChange>(changeFile);
Log.LogInformation("Uploading change to OSM");
var changeTags = GetCommitTags(description);
changeTags.RemoveKey("municipality");
var osmApiClient = new BasicAuthClient(Static.HttpClient,
Static.LogFactory.CreateLogger <BasicAuthClient>(), Static.Config["OsmApiUrl"],
Static.Config["OsmUsername"], Static.Config["OsmPassword"]);
var changeParts = change.SplitByCount().ToArray();
var result = new long[changeParts.Length];
int i = 0;
foreach (var part in changeParts)
{
if (changeParts.Length > 1)
{
changeTags.AddOrReplace("change-part", $"part {i + 1} of {changeParts.Length}");
}
result[i] = await UploadChangePart(part, changeTags, osmApiClient, $"{description}/Uploaded/");
i++;
}
return(result);
}
private static NetTopologySuite.Geometries.Geometry LoadBounds(string scopeName, long relationId)
{
var scopeFile = $"{scopeName}\\Scope.osm";
var osm = FileSerializer.ReadXml <Osm>(scopeFile);
CompleteRelation relation;
if (osm != null)
{
relation = osm.GetElements().ToComplete().OfType <CompleteRelation>().First();
}
else
{
NonAuthClient client = new NonAuthClient("https://www.osm.org/api/", new HttpClient());
relation = client.GetCompleteRelation(relationId).Result;
FileSerializer.WriteXml(scopeFile, relation.ToSimpleWithChildren().AsOsm());
}
var geometry = FeatureInterpreter.DefaultInterpreter.Interpret(relation).First().Geometry;
if (geometry is LinearRing linRing)
{
var polygon = Polygon.DefaultFactory.CreatePolygon(linRing); // Or multipolygon?
return(polygon);
}
else if (geometry is MultiPolygon multi)
{
return(multi);
//Polygon.DefaultFactory.CreateMultiPolygon();
}
throw new Exception("unexpected geometry type: " + geometry.GetType().Name);
}
public static void Review(string scopeName)
{
var review = FileSerializer.ReadXml <Osm>(scopeName + "/Conflated.Review.osm");
if (review != null)
{
foreach (var element in review.GetElements().Where(e => e.Tags != null))
{
foreach (var revTag in element.Tags.Where(t => t.Key == WarnKey || t.Key == ErrorKey))
{
Console.WriteLine(Identify(element) + " " + revTag);
}
}
}
}
private static IEnumerable <OsmGeo> OpenFile(string path)
{
var extension = Path.GetExtension(path);
if (extension.Equals(".pbf", StringComparison.OrdinalIgnoreCase))
{
// Warning: IDisposable not disposed.
return(new OsmSharp.Streams.PBFOsmStreamSource(new FileInfo(path).OpenRead()));
}
else if (extension.Equals(".osm", StringComparison.OrdinalIgnoreCase))
{
return(FileSerializer.ReadXml <Osm>(path).GetElements());
}
throw new NotImplementedException(extension);
}
public static IEnumerable <Tuple <OsmChange, DiffResult> > GetAllChangeFiles()
{
Static.Municipalities = FileSerializer.ReadJson <Dictionary <string, GeoJsonAPISource.Municipality> >("MaineMunicipalities.json");
foreach (var municipality in Static.Municipalities.Values)
{
foreach (var changeId in municipality.ChangeSetIds)
{
var changePath = $"{municipality.Name}/Uploaded/{changeId}-Conflated.osc";
var change = FileSerializer.ReadXml <OsmChange>(changePath);
var diffPath = $"{municipality.Name}/Uploaded/{changeId}-DiffResult.diff";
var diff = FileSerializer.ReadXml <DiffResult>(diffPath);
yield return(Tuple.Create(change, diff));
}
}
}
private static Osm TranslateAndDisolve(string from, NetTopologySuite.Geometries.Geometry scope = null)
{
var osm = FileSerializer.ReadXml <Osm>(from);
var index = OsmSharp.Db.Impl.Extensions.CreateSnapshotDb(new MemorySnapshotDb(osm.GetElements()));
var translated = Translate(osm);
ReverseOneWays(translated);
var disolved = Geometry.Disolve(translated, "medot:objectid");
var scoped = disolved.Where(e => scope == null || FeatureInterpreter.DefaultInterpreter.Interpret(e).First().Geometry.Intersects(scope)).ToArray();
// Hide the object UIDs in User, away from the tags.
foreach (var e in scoped)
{
e.UserName = e.Tags["medot:objectid"];
e.Tags.RemoveKey("medot:objectid");
}
return(scoped.AsOsm(index));
}
public static void RecreateMissingDiffResultFiles()
{
Static.Municipalities = FileSerializer.ReadJson <Dictionary <string, GeoJsonAPISource.Municipality> >("MaineMunicipalities.json");
var osmApiClient = new NonAuthClient(Static.Config["OsmApiUrl"], Static.HttpClient,
Static.LogFactory.CreateLogger <NonAuthClient>());
foreach (var municipality in Static.Municipalities.Values.Where(m => m.ChangeSetIds.Any(id => id != -1)))
{
foreach (var changeId in municipality.ChangeSetIds)
{
var diffPath = $"{municipality.Name}/Uploaded/{changeId}-DiffResult.diff";
if (!File.Exists(diffPath))
{
Console.WriteLine(diffPath);
var changePath = $"{municipality.Name}/Uploaded/{changeId}-Conflated.osc";
var change = FileSerializer.ReadXml <OsmChange>(changePath);
var mine = change.Create.OfType <Node>().ToArray();
var theirs = osmApiClient.GetChangesetDownload(changeId).Result;
List <OsmGeoResult> results = theirs.Modify.Select(e => e.AsDiffResult()).ToList();
var map = Geometry.NodesInOrNearCompleteElements(theirs.Create.OfType <Node>().ToArray(), mine, 0, 0, new HashSet <long>());
if (map.Count != theirs.Create.Length || map.Any(pair => pair.Value.Count(v => Tags.AreEqual(v.Tags, pair.Key.Tags)) != 1))
{
throw new Exception("bad map");
}
results.AddRange(map.Select(pair => pair.Value.Single(v => Tags.AreEqual(v.Tags, pair.Key.Tags)).AsDiffResult(pair.Key.Id, 1)));
var diffResult = new DiffResult()
{
Version = 0.6, Generator = "OsmPipeline", Results = results.ToArray()
};
FileSerializer.WriteXml(diffPath, diffResult);
}
}
}
}
// nohousenumber=yes
// consider just leaving out all addr:unit=*
// option to moveNode or not. (Do you trust E911 locations more than OSM?)
// MatchWidth list to handle multi-match
// Maybe fix addr:street punctuation on elements that I didn't add or update
// Maybe fix addr:street should be addr:place on elements that I didn't add or update
// Maybe fix nodes merging into buildings on elements that I didn't add or update
public void Main()
{
Static.Municipalities = FileSerializer.ReadJsonCacheOrSource("MaineMunicipalities.json",
GetMunicipalities).Result;
ShowProgress();
Municipality = Static.Municipalities.Values.First(m => !m.ChangeSetIds.Any()).Name;
Console.WriteLine("Starting in " + Municipality);
while (true)
{
Console.Write("> ");
var userInput = Console.ReadLine();
if (Is(userInput, "reference"))
{
//Func<Feature, bool> filter = f => (f.Geometry as Point).Coordinates.Longitude >= -70.505;
var reference = References.Fetch(Municipality).Result;
FileSerializer.WriteXml(Municipality + "/Reference.osm", reference);
File.Delete(Municipality + "/Conflated.osc");
}
else if (Is(userInput, "review ref"))
{
var reference = GetReference();
References.Report(reference.GetElements().ToArray());
}
else if (Is(userInput, "subject"))
{
var reference = GetReference();
var subject = Subjects.GetElementsInBoundingBox(reference.Bounds.ExpandBy(15));
FileSerializer.WriteXml(Municipality + "/Subject.osm", subject);
File.Delete(Municipality + "/Conflated.osc");
Console.WriteLine("ChangeId high watermark: " + subject.GetHighestChangeSetId());
}
else if (Is(userInput, "conflate"))
{
DoConflate();
}
else if (Is(userInput, "review con"))
{
Conflate.Review(Municipality);
}
else if (Is(userInput, "review"))
{
OpenJosm();
}
else if (Is(userInput, "list"))
{
var key = userInput.Split(" ")[1];
var reference = GetReference();
var values = reference.GetElements().Where(e => e.Tags.ContainsKey(key)).Select(e => "\n\t" + e.Tags[key]).GroupBy(n => n).ToArray();
Console.WriteLine(string.Concat(values.Select(v => v.Key + "\tx" + v.Count())));
}
else if (Is(userInput, "filter"))
{
var key = userInput.Split(" ")[1];
var reference = GetReference();
DoFilter(key, reference.GetElements());
File.Delete(Municipality + "/Reference.osm");
File.Delete(Municipality + "/Conflated.osc");
}
else if (Is(userInput, "note"))
{
Static.Municipalities[Municipality].Notes += "/n" + userInput.Split(' ', 2)[1];
FileSerializer.WriteJson("MaineMunicipalities.json", Static.Municipalities);
}
else if (Is(userInput, "WhiteAll"))
{
var review = FileSerializer.ReadXml <Osm>(Municipality + "/Conflated.Review.osm");
var selection = review.GetElements()
.Where(e => e.Tags != null && e.Tags.ContainsKey(Static.maineE911id))
.Select(e => e.Tags[Static.maineE911id])
.SelectMany(id => id.Split(new char[] { ' ', ',', ';', '-' }, StringSplitOptions.RemoveEmptyEntries))
.Select(long.Parse)
.Except(Static.Municipalities[Municipality].WhiteList)
.ToArray();
Static.Municipalities[Municipality].WhiteList.AddRange(selection);
FileSerializer.WriteJson("MaineMunicipalities.json", Static.Municipalities);
File.Delete(Municipality + "/Conflated.osc");
}
else if (Is(userInput, "white"))
{
var selection = userInput.Split(' ', 2)[1]
.Split(new char[] { ' ', ',', ';', '-', '=' }, StringSplitOptions.RemoveEmptyEntries)
.Where(c => long.TryParse(c, out _))
.Select(long.Parse)
.Except(Static.Municipalities[Municipality].WhiteList)
.ToArray();
Static.Municipalities[Municipality].WhiteList.AddRange(selection);
FileSerializer.WriteJson("MaineMunicipalities.json", Static.Municipalities);
File.Delete(Municipality + "/Conflated.osc");
}
else if (Is(userInput, "blacktag"))
{
var tag = userInput.Split(' ', 2)[1].Replace("maineE911id=", "");
Static.Municipalities[Municipality].BlackTags.Add(tag);
Static.Municipalities[Municipality].BlackTags = Static.Municipalities[Municipality].BlackTags.Distinct().ToList();
FileSerializer.WriteJson("MaineMunicipalities.json", Static.Municipalities);
File.Delete(Municipality + "/Reference.osm");
File.Delete(Municipality + "/Conflated.osc");
}
else if (Is(userInput, "black"))
{
AddToList(userInput.Split(' ', 2)[1], Static.Municipalities[Municipality].BlackList);
FileSerializer.WriteJson("MaineMunicipalities.json", Static.Municipalities);
File.Delete(Municipality + "/Reference.osm");
File.Delete(Municipality + "/Conflated.osc");
}
else if (Is(userInput, "ignore"))
{
AddToList(userInput.Split(' ', 2)[1], Static.Municipalities[Municipality].IgnoreList);
FileSerializer.WriteJson("MaineMunicipalities.json", Static.Municipalities);
}
else if (Is(userInput, "commit"))
{
var change = FileSerializer.ReadXml <OsmChange>(Municipality + "/Conflated.osc");
var results = Subjects.UploadChange(change, Municipality).Result;
Static.Municipalities[Municipality].ChangeSetIds.AddRange(results);
Static.Municipalities[Municipality].ImportDate = DateTime.UtcNow;
FileSerializer.WriteJson("MaineMunicipalities.json", Static.Municipalities);
Console.WriteLine("Finished!");
Next();
}
else if (Is(userInput, "skip"))
{
Static.Municipalities[Municipality].Notes += " SKIPPED";
Static.Municipalities[Municipality].ChangeSetIds.Add(-1);
FileSerializer.WriteJson("MaineMunicipalities.json", Static.Municipalities);
}
else if (Is(userInput, "next"))
{
Next();
}
else if (Is(userInput, "switch"))
{
Municipality = ChooseMunicipality();
ShowProgress();
}
else if (Is(userInput, "folder"))
{
OpenExplorer();
}
else if (Is(userInput, "remind"))
{
var parts = userInput.Split(' ', 3);
var id = long.Parse(parts[1].Replace("maineE911id=", ""));
var reference = GetReference();
var element = reference.Nodes.First(e => Math.Abs(e.Id.Value) == Math.Abs(id));
var message = parts.Length > 2 ? parts[2] : "The addresses imported on this neighborhood need to be aligned with the correct buildings";
Subjects.CreateNote(element.Latitude.Value, element.Longitude.Value, message).Wait();
}
else if (Is(userInput, "help"))
{
Console.WriteLine("Options:");
Console.WriteLine("\tSwitch");
Console.WriteLine("\tNext");
Console.WriteLine("\tReference");
Console.WriteLine("\tReview Reference");
Console.WriteLine("\tSubject");
Console.WriteLine("\tConflate");
Console.WriteLine("\tReview Conflate");
Console.WriteLine("\tReview");
Console.WriteLine("\tList [key]");
Console.WriteLine("\tFilter [key]");
Console.WriteLine("\t\t[Y/N/AUnit/Descr/Build/Move]");
Console.WriteLine("\tNote [message]");
Console.WriteLine("\tWhitelist [###]<,[###]...>");
Console.WriteLine("\tWhiteAll");
Console.WriteLine("\tBlacklist [###]<,[###]...>");
Console.WriteLine("\tBlackTag [###].[key] || *.[key]=[value]");
Console.WriteLine("\tIgnore [###]<,[###]...>");
Console.WriteLine("\tRemind [ref###] <message>");
Console.WriteLine("\tCommit");
}
else
{
Console.WriteLine("What?");
}
Console.WriteLine("Done");
}
}