// est: since going from 6 to 10 took 1 minute, we might expect doing all 256 would take 256 minutes // if we break it up into quadrants using the same library, maybe it'll only take (4+1+1/16...) roughly 5.33 minutes? // actually took 8.673 mins (went from 450MB to 455MB) // estimated time to segment the whole 43.1 GB planet? 12/28/2018 = 8.673 * 43.1 / 8.05 * 47.7833 = 36.98 hours public static void BreakupFile(string filePath, ISector sector, int targetZoom) { if (sector.Zoom == targetZoom) { return; } List <ISector> quadrants = sector.GetChildrenAtLevel(sector.Zoom + 1); if (READ_BREAKUP_STEP <= CURRENT_BREAKUP_STEP) { foreach (var quadrant in quadrants) { String quadrantPath = OSMPaths.GetSectorPath(quadrant); if (File.Exists(quadrantPath)) { File.Delete(quadrantPath); // we're assuming it's corrupted } if (!Directory.Exists(Path.GetDirectoryName(quadrantPath))) { Directory.CreateDirectory(Path.GetDirectoryName(quadrantPath)); } var fInfo = new FileInfo(filePath); using (var fileInfoStream = fInfo.OpenRead()) { using (var source = new PBFOsmStreamSource(fileInfoStream)) { var filtered = source.FilterNodes(x => x.Longitude.HasValue && x.Latitude.HasValue && quadrant.ContainsLongLat(new LongLat(x.Longitude.Value * Math.PI / 180, x.Latitude.Value * Math.PI / 180)), true); using (var stream = new FileInfo(quadrantPath).Open(FileMode.Create, FileAccess.ReadWrite)) { var target = new PBFOsmStreamTarget(stream, true); target.RegisterSource(filtered); target.Pull(); target.Close(); } } } } if (Path.GetFileName(filePath).ToLower() != Path.GetFileName(OSMPaths.GetPlanetPath()).ToLower()) { File.Delete(filePath); } } BreakupStepDone(); foreach (var quadrant in quadrants) { String quadrantPath = OSMPaths.GetSectorPath(quadrant); BreakupFile(quadrantPath, quadrant, targetZoom); } }
static int READ_BREAKUP_STEP = 0; // easy way to allow continuation, should usually equal (#filesThatArenttoplevel)/3 public static void SegmentOSMPlanet() { READ_BREAKUP_STEP = int.Parse(File.ReadAllText(OSMPaths.GetPlanetStepPath()).Split(',')[0]); // file should contain the number of physical breakups that were finished List <ISector> quadrants = ZCoords.GetSectorManager().GetTopmostOSMSectors(); if (READ_BREAKUP_STEP <= CURRENT_BREAKUP_STEP) { foreach (var quadrant in quadrants) { String quadrantPath = OSMPaths.GetSectorPath(quadrant); if (File.Exists(quadrantPath)) { File.Delete(quadrantPath); // we're assuming it's corrupted } if (!Directory.Exists(Path.GetDirectoryName(quadrantPath))) { Directory.CreateDirectory(Path.GetDirectoryName(quadrantPath)); } var fInfo = new FileInfo(OSMPaths.GetPlanetPath()); using (var fileInfoStream = fInfo.OpenRead()) { using (var source = new PBFOsmStreamSource(fileInfoStream)) { var filtered = source.FilterNodes(x => x.Longitude.HasValue && x.Latitude.HasValue && quadrant.ContainsLongLat(new LongLat(x.Longitude.Value * Math.PI / 180, x.Latitude.Value * Math.PI / 180)), true); using (var stream = new FileInfo(quadrantPath).Open(FileMode.Create, FileAccess.ReadWrite)) { var target = new PBFOsmStreamTarget(stream, true); target.RegisterSource(filtered); target.Pull(); target.Close(); } } } } } BreakupStepDone(); foreach (var quadrant in quadrants) { String quadrantPath = OSMPaths.GetSectorPath(quadrant); BreakupFile(quadrantPath, quadrant, ZCoords.GetSectorManager().GetHighestOSMZoom()); } }