static void CacheTest()
{
long[] relIds =
{
418716L, // Herrnhut
408308L, // Berlin Weißensee
62422L, // Berlin
62504L, // Brandenburg
51477L, // Deutschland
3920249L, // Portuagl - Aveiro
3459013L // Portugal - Porto
};
using (var pbf = new OsmPbfReader(PbfPath))
using (var cache = new OsmCache(pbf))
{
foreach (long relId in relIds)
{
OsmRelation relation;
OsmWay[] ways;
OsmNode[] nodes;
if (!cache.ReadRelation(relId, out relation, out ways, out nodes))
{
throw new Exception("relation not found: " + relId);
}
// --- validation ---
//OsmRelation relation2;
//OsmWay[] ways2;
//OsmNode[] nodes2;
//if (!cache.ReadRelationDirect(relId, out relation2, out ways2, out nodes2))
//{
// throw new Exception("relation not found: " + relId);
//}
//if (relation.ToString() != relation2.ToString()) throw new Exception("meow");
//if (string.Join(",", relation.members.Select(m => m.ToString())) != string.Join(",", relation2.members.Select(m => m.ToString()))) throw new Exception("meow");
//if (string.Join(",", relation.values.Select(v => v.Key + "-" + v.Value)) != string.Join(",", relation2.values.Select(v => v.Key + "-" + v.Value))) throw new Exception("meow");
//if (ways.Length != ways2.Length) throw new Exception("meow");
//for (int i = 0; i < ways.Length; i++)
//{
// if (ways[i].ToString() != ways2[i].ToString()) throw new Exception("meow");
// if (string.Join(",", ways[i].values.Select(v => v.Key + "-" + v.Value)) != string.Join(",", ways2[i].values.Select(v => v.Key + "-" + v.Value))) throw new Exception("meow");
// if (string.Join(",", ways[i].nodeIds) != string.Join(",", ways2[i].nodeIds)) throw new Exception("meow");
//}
//if (nodes.Length != nodes2.Length) throw new Exception("meow");
//for (int i = 0; i < nodes.Length; i++)
//{
// if (nodes[i].ToString() != nodes2[i].ToString()) throw new Exception("meow");
// if (string.Join(",", nodes[i].values.Select(v => v.Key + "-" + v.Value)) != string.Join(",", nodes2[i].values.Select(v => v.Key + "-" + v.Value))) throw new Exception("meow");
//}
}
}
}
static void SpeedCheck()
{
using (var pbf = new OsmPbfReader(PbfPath))
{
long count = 0;
long totalCount = pbf.nodeIndex.Sum(x => x.nodeCount);
//foreach (var node in pbf.ReadAllNodes())
//{
// count++;
// if ((count & 1048575) == 1)
// {
// Console.WriteLine(count.ToString("N0") + " / " + totalCount.ToString("N0"));
// }
//}
var blockTimes = new List <long>();
var tim = Stopwatch.StartNew();
foreach (var nodes in pbf.ReadAllNodes4())
{
blockTimes.Add(tim.ElapsedMilliseconds);
tim.Restart();
count += nodes.Length;
lock (OsmPbfReader.times)
{
Console.WriteLine(count.ToString("N0") + " / " + totalCount.ToString("N0") + " - " + OsmPbfReader.times.Average().ToString("N2") + " / " + blockTimes.Average().ToString("N2"));
}
//foreach (var node in nodes)
//{
// count++;
// if ((count & 1048575) == 1)
// {
// Console.WriteLine(count.ToString("N0") + " / " + totalCount.ToString("N0"));
// }
//}
//if (count > 50000000)
//{
// break;
//}
nodes.Dispose();
}
}
}
public OsmCache(OsmPbfReader pbf)
{
if (!File.Exists(CacheIndexFile))
{
File.WriteAllBytes(CacheIndexFile, new byte[0]);
}
if (!File.Exists(CacheDataFile))
{
File.WriteAllBytes(CacheDataFile, new byte[1]);
}
// --- Read Index ---
relationsIndex = new Dictionary <long, long>();
waysIndex = new Dictionary <long, long>();
nodesIndex = new Dictionary <long, long>();
var buf = File.ReadAllBytes(CacheIndexFile);
for (int p = 0; p < buf.Length;)
{
var type = (MemberType)buf[p++];
ulong id, index;
p += ProtoBuf.ReadVarInt(buf, p, out id);
p += ProtoBuf.ReadVarInt(buf, p, out index);
switch (type)
{
case MemberType.Relation: relationsIndex.Add((long)id, (long)index); break;
case MemberType.Way: waysIndex.Add((long)id, (long)index); break;
case MemberType.Node: nodesIndex.Add((long)id, (long)index); break;
default: throw new Exception("unknown type?");
}
}
cacheData = new FileStream(CacheDataFile, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
pbfReader = pbf;
}
static void SearchRelations()
{
var found = new List <OsmRelation>();
using (var pbf = new OsmPbfReader(PbfPath))
{
long pos = 0;
long relTotalCount = pbf.relationIndex.Sum(x => x.relationCount);
foreach (var rel in pbf.ReadAllRelations())
{
if ((pos & 0xfff) == 0)
{
Console.WriteLine("{0:N0} / {1:N0}", pos, relTotalCount);
}
pos++;
if (rel.values.Any(x => x.Key == "place" && x.Value == "sea"))
{
string deName = rel.values.FirstOrDefault(x => x.Key == "name:de").Value ?? "xx";
Console.WriteLine("found: " + deName);
found.Add(rel);
}
}
}
}
static void ConverterTest_1_ExtractNodes()
{
Directory.CreateDirectory("../tmp");
using (var wdatRaw = File.Create("../tmp/node_rawfull.dat"))
using (var wdatRawIndex = File.Create("../tmp/node_rawfull_index.dat"))
using (var pbf = new OsmPbfReader(PbfPath))
{
long cc = 0;
var rawBuf = new byte[4096];
int rawLen = 0;
long rawId = 0;
long rawPos = 0;
var buf = new byte[2000 * 1048576];
int len = 0;
long lastId = 0;
long lastPos = 0;
int block = 0;
long totalPos = 0;
long totalSum = pbf.nodeIndex.Sum(x => x.nodeCount);
wdatRawIndex.Write(BitConverter.GetBytes(0L), 0, sizeof(long));
wdatRawIndex.Write(BitConverter.GetBytes(0L), 0, sizeof(long));
foreach (var node in pbf.ReadAllNodes())
{
totalPos++;
var gpsPos = new GpsPos(node);
rawLen += ProtoBuf.WriteVarInt(rawBuf, rawLen, ProtoBuf.UnsignedInt64(node.id - rawId));
rawId = node.id;
long nextFullPos = gpsPos.Int64Pos;
rawLen += ProtoBuf.WriteVarInt(rawBuf, rawLen, ProtoBuf.UnsignedInt64(nextFullPos - rawPos));
rawPos = nextFullPos;
if (rawLen > rawBuf.Length - 18)
{
while (rawLen < rawBuf.Length)
{
rawBuf[rawLen++] = 0;
}
wdatRaw.Write(rawBuf, 0, rawBuf.Length);
wdatRawIndex.Write(BitConverter.GetBytes(rawId + 1), 0, sizeof(long));
wdatRawIndex.Write(BitConverter.GetBytes(wdatRaw.Length), 0, sizeof(long));
rawLen = 0;
rawId = 0;
rawPos = 0;
}
//todo
//if (node.values.Length > 0)
//{
// cc++;
// if ((ushort)cc == 0) Console.WriteLine(cc.ToString("N0") + " (" + (100.0 / totalSum * totalPos).ToString("N2") + " %) - " + (len / 1048576.0).ToString("N1") + " MByte / " + (wdatRaw.Length / 1048576.0).ToString("N1") + " MByte");
// len += ProtoBuf.WriteVarInt(buf, len, ProtoBuf.UnsignedInt64(node.id - lastId));
// lastId = node.id;
// long nextPos = gpsPos.Int64Pos;
// len += ProtoBuf.WriteVarInt(buf, len, ProtoBuf.UnsignedInt64(nextPos - lastPos));
// lastPos = nextPos;
// len += ProtoBuf.WriteVarInt(buf, len, (uint)node.values.Length);
// foreach (var v in node.values)
// {
// len += ProtoBuf.WriteString(buf, len, v.Key);
// len += ProtoBuf.WriteString(buf, len, v.Value);
// }
// if (len > buf.Length - 65536)
// {
// block++;
// using (var wdat = File.Create("../tmp/node_block_" + block + "_" + lastId + ".dat"))
// {
// wdat.Write(buf, 0, len);
// len = 0;
// lastId = 0;
// lastPos = 0;
// }
// }
//}
}
if (rawLen > 0)
{
while (rawLen < rawBuf.Length)
{
rawBuf[rawLen++] = 0;
}
wdatRaw.Write(rawBuf, 0, rawBuf.Length);
}
if (len > 0)
{
block++;
using (var wdat = File.Create("../tmp/node_block_" + block + "_" + lastId + ".dat"))
{
wdat.Write(buf, 0, len);
}
}
//foreach (var way in pbf.ReadAllWays())
//{
// Console.WriteLine(way);
//}
//foreach (var relation in pbf.ReadAllRelations())
//{
// cc++;
// if ((byte)cc == 0)
// {
// Console.WriteLine(cc.ToString("N0") + " - " + relation);
// }
//}
}
}
static void ParseTest()
{
using (var pbf = new OsmPbfReader(PbfPath))
{
//long testRelationID = 418713; // Herrnhut (Germany)
//long testRelationID = 62422; // Berlin
//long testRelationID = 62504; // Brandenburg (Germany) - https://www.openstreetmap.org/relation/62504
//long testRelationID = 1434381; // Insel Rügen (Germany)
long testRelationID = 51477; // Germany
//long testRelationID = 2214684; // La Gomera (Canarias)
//long testRelationID = 2182003; // Menorca (Balearic Islands)
//long testRelationID = 11775386; // La Palma (Canarias)
var relations = pbf.ReadRelations(testRelationID);
//var relations = pbf.ReadRelations(418713, 62422, 62504, 1434381, 51477, 2214684, 2182003, 11775386);
for (; ;) // untergeordnete Relationen einlesen
{
var needRelations = new List <long>();
foreach (var r in relations)
{
foreach (var sub in r.members.Where(x => x.type == MemberType.Relation))
{
if (relations.Any(x => x.id == sub.id))
{
continue;
}
needRelations.Add(sub.id);
}
}
if (needRelations.Count == 0)
{
break;
}
relations = relations.Concat(pbf.ReadRelations(needRelations.ToArray())).ToArray(relations.Length + needRelations.Count);
break;
}
var ways = pbf.ReadWays(relations.SelectMany(r => r.members.Where(x => x.type == MemberType.Way).Select(x => x.id)).ToArray());
Array.Sort(ways, (x, y) => x.id.CompareTo(y.id));
var nodes = pbf.ReadNodes(relations.SelectMany(r => r.members.Where(x => x.type == MemberType.Node).Select(x => x.id)).Concat(ways.SelectMany(w => w.nodeIds)).ToArray());
Array.Sort(nodes, (x, y) => x.id.CompareTo(y.id));
Console.WriteLine("nodes: {0:N0}", nodes.Length);
var knownNodes = new HashSet <long>();
Func <StreamWriter, long, string, int> writePath = null;
writePath = (wdat, relationId, type) =>
{
int count = 0;
var rel = relations.FirstOrDefault(x => x.id == relationId);
if (rel.id == 0)
{
return(0);
}
foreach (var member in rel.members)
{
switch (member.type)
{
case MemberType.Node: break; // skip
case MemberType.Way:
{
if (member.role == type)
{
var way = ways.BinarySearchSingle(w => w.id - member.id);
foreach (var nodeId in way.nodeIds)
{
if (knownNodes.Contains(nodeId))
{
continue;
}
knownNodes.Add(nodeId);
var node = nodes.BinarySearchSingle(n => n.id - nodeId);
wdat.WriteLine(node.latCode + "," + node.lonCode);
count++;
}
}
} break;
case MemberType.Relation: count += writePath(wdat, member.id, type); break;
}
}
return(count);
};
using (var wdat = new StreamWriter(testRelationID + "_outer.txt"))
{
writePath(wdat, testRelationID, "outer");
}
using (var wdat = new StreamWriter(testRelationID + "_inner.txt"))
{
writePath(wdat, testRelationID, "inner");
}
}
}
static void MappingTest()
{
//long[] relIds = { 3920249L };
//long[] relIds = { 3459013L };
long[] relIds = { 62504L }; // Brandenburg
//long[] relIds = { 51477L }; // Deutschland
using (var pbf = new OsmPbfReader(PbfPath))
{
foreach (long relId in relIds)
{
OsmNode[] nodesPath;
if (File.Exists("path_cache_" + relId + ".dat"))
{
byte[] buf = File.ReadAllBytes("path_cache_" + relId + ".dat");
int len = ReadNodesPath(buf, 0, out nodesPath);
if (len != buf.Length)
{
throw new IOException();
}
}
else
{
var rels = pbf.ReadRelations(relIds);
var ways = pbf.ReadWays(rels.First().members.Where(x => x.type == MemberType.Way).Select(x => x.id).ToArray());
var nodes = pbf.ReadNodes(ways.SelectMany(x => x.nodeIds).ToArray());
nodesPath = GetPath(ways, nodes).ToArray();
byte[] buf = new byte[1048576 * 256];
int len = WriteNodesPath(buf, 0, nodesPath);
Array.Resize(ref buf, len);
File.WriteAllBytes("path_cache_" + relId + ".dat", buf);
}
var polyLines = new List <GpsLine>();
for (int i = 1; i < nodesPath.Length; i++)
{
if (nodesPath[i - 1].latCode >= nodesPath[i].latCode)
{
polyLines.Add(new GpsLine(new GpsPos(nodesPath[i - 1]), new GpsPos(nodesPath[i])));
}
else
{
polyLines.Add(new GpsLine(new GpsPos(nodesPath[i]), new GpsPos(nodesPath[i - 1])));
}
}
polyLines.Sort((x, y) => x.pos1.posY.CompareTo(y.pos1.posY));
// --- init ---
int maxLinesPerStripe = (int)(Math.Sqrt(nodesPath.Length) * 0.3) + 1;
maxLinesPerStripe = 0;
int limitStripes = polyLines.GroupBy(line => line.pos1.posY).Max(g => g.Count());
if (maxLinesPerStripe < limitStripes)
{
maxLinesPerStripe = limitStripes;
}
var firstLines = new List <GpsLine>();
foreach (var line in polyLines)
{
if (firstLines.Count >= maxLinesPerStripe && firstLines.Last().pos1.posY < line.pos1.posY)
{
break;
}
firstLines.Add(line);
}
var stripes = new List <Tuple <uint, uint, List <GpsLine> > >
{
new Tuple <uint, uint, List <GpsLine> >(firstLines.First().pos1.posY, firstLines.Last().pos1.posY, firstLines)
};
for (; ;)
{
uint startY = stripes.Last().Item2;
var nextLines = polyLines.Where(line => line.pos1.posY < startY && line.pos2.posY >= startY).ToList();
int maxCount = nextLines.Count + maxLinesPerStripe;
foreach (var line in polyLines)
{
if (line.pos1.posY >= startY)
{
if (nextLines.Count > maxCount && nextLines.Last().pos1.posY < line.pos1.posY)
{
break;
}
nextLines.Add(line);
}
}
stripes.Add(new Tuple <uint, uint, List <GpsLine> >(nextLines.First().pos1.posY, nextLines.Last().pos1.posY, nextLines));
if (Equals(nextLines.Last(), polyLines.Last()))
{
break;
}
}
int sumStripes1 = stripes.Sum(s => s.Item3.Count);
stripes.Add(new Tuple <uint, uint, List <GpsLine> >(stripes.Last().Item2 + 1, stripes.Last().Item2, new List <GpsLine>()));
var fastStripes = stripes.Select((x, i) => new GpsStripe(i == 0 ? x.Item1 : stripes[i - 1].Item2, x.Item2, x.Item3.ToArray())).ToArray();
DrawTest(nodesPath, polyLines, fastStripes);
}
}
}
