void WriteIndex(MemberType type, long id, long index)
{
switch (type)
{
case MemberType.Relation: relationsIndex.Add(id, index); break;
case MemberType.Way: waysIndex.Add(id, index); break;
case MemberType.Node: nodesIndex.Add(id, index); break;
default: throw new Exception("unknown type?");
}
byte[] miniBuf = new byte[64];
int p = 0;
miniBuf[p] = (byte)type; p++;
p += ProtoBuf.WriteVarInt(miniBuf, p, (ulong)id);
p += ProtoBuf.WriteVarInt(miniBuf, p, (ulong)index);
using (var indexFile = File.OpenWrite(CacheIndexFile))
{
indexFile.Position = indexFile.Length;
indexFile.Write(miniBuf, 0, p);
}
}
long WriteRelationCache(OsmRelation osmRelation, OsmWay[] osmWays, OsmNode[] osmNodes)
{
long relationIndex = cacheData.Length;
int p = sizeof(int); // cacheblock-size
var buf = cacheBuffer;
p += osmRelation.WriteBinary(buf, p);
p += ProtoBuf.WriteVarInt(buf, p, (ulong)osmWays.Length);
var tmpWays = osmWays.ToArray();
EncodeDelta(tmpWays);
foreach (var way in tmpWays)
{
p += way.WriteBinary(buf, p);
}
p += ProtoBuf.WriteVarInt(buf, p, (ulong)osmNodes.Length);
var tmpNodes = osmNodes.ToArray();
EncodeDelta(tmpNodes);
foreach (var node in tmpNodes)
{
p += node.WriteBinary(buf, p);
}
buf[0] = (byte)p;
buf[1] = (byte)(p >> 8);
buf[2] = (byte)(p >> 16);
buf[3] = (byte)(p >> 24);
cacheData.Position = relationIndex;
cacheData.Write(buf, 0, p);
return(relationIndex);
}
public int WriteBinary(byte[] buf, int ofs)
{
int p = 0;
p += ProtoBuf.WriteVarInt(buf, ofs + p, (ulong)id);
buf[ofs + p] = (byte)type; p++;
p += ProtoBuf.WriteString(buf, ofs + p, role);
return(p);
}
static int WriteNodesPath(byte[] buf, int ofs, OsmNode[] nodes)
{
int p = ProtoBuf.WriteVarInt(buf, ofs, (uint)nodes.Length);
foreach (var node in nodes)
{
p += node.WriteBinary(buf, ofs + p);
}
return(p);
}
public int WriteBinary(byte[] buf, int ofs)
{
int p = 0;
p += ProtoBuf.WriteVarInt(buf, ofs + p, (ulong)id);
p += ProtoBuf.WriteVarInt(buf, ofs + p, ProtoBuf.UnsignedInt32(latCode));
p += ProtoBuf.WriteVarInt(buf, ofs + p, ProtoBuf.UnsignedInt32(lonCode));
p += ProtoBuf.WriteVarInt(buf, ofs + p, (uint)values.Length);
foreach (var val in values)
{
p += ProtoBuf.WriteString(buf, ofs + p, val.Key);
p += ProtoBuf.WriteString(buf, ofs + p, val.Value);
}
return(p);
}
public int WriteBinary(byte[] buf, int ofs)
{
int p = 0;
p += ProtoBuf.WriteVarInt(buf, ofs + p, (ulong)id);
p += ProtoBuf.WriteVarInt(buf, ofs + p, (uint)values.Length);
foreach (var val in values)
{
p += ProtoBuf.WriteString(buf, ofs + p, val.Key);
p += ProtoBuf.WriteString(buf, ofs + p, val.Value);
}
p += ProtoBuf.WriteVarInt(buf, ofs + p, (uint)members.Length);
foreach (var member in members)
{
p += member.WriteBinary(buf, ofs + p);
}
return(p);
}
static void ConverterTest_1_ExtractNodes()
{
Directory.CreateDirectory("../tmp");
using (var wdatRaw = File.Create("../tmp/node_rawfull.dat"))
using (var wdatRawIndex = File.Create("../tmp/node_rawfull_index.dat"))
using (var pbf = new OsmPbfReader(PbfPath))
{
long cc = 0;
var rawBuf = new byte[4096];
int rawLen = 0;
long rawId = 0;
long rawPos = 0;
var buf = new byte[2000 * 1048576];
int len = 0;
long lastId = 0;
long lastPos = 0;
int block = 0;
long totalPos = 0;
long totalSum = pbf.nodeIndex.Sum(x => x.nodeCount);
wdatRawIndex.Write(BitConverter.GetBytes(0L), 0, sizeof(long));
wdatRawIndex.Write(BitConverter.GetBytes(0L), 0, sizeof(long));
foreach (var node in pbf.ReadAllNodes())
{
totalPos++;
var gpsPos = new GpsPos(node);
rawLen += ProtoBuf.WriteVarInt(rawBuf, rawLen, ProtoBuf.UnsignedInt64(node.id - rawId));
rawId = node.id;
long nextFullPos = gpsPos.Int64Pos;
rawLen += ProtoBuf.WriteVarInt(rawBuf, rawLen, ProtoBuf.UnsignedInt64(nextFullPos - rawPos));
rawPos = nextFullPos;
if (rawLen > rawBuf.Length - 18)
{
while (rawLen < rawBuf.Length)
{
rawBuf[rawLen++] = 0;
}
wdatRaw.Write(rawBuf, 0, rawBuf.Length);
wdatRawIndex.Write(BitConverter.GetBytes(rawId + 1), 0, sizeof(long));
wdatRawIndex.Write(BitConverter.GetBytes(wdatRaw.Length), 0, sizeof(long));
rawLen = 0;
rawId = 0;
rawPos = 0;
}
//todo
//if (node.values.Length > 0)
//{
// cc++;
// if ((ushort)cc == 0) Console.WriteLine(cc.ToString("N0") + " (" + (100.0 / totalSum * totalPos).ToString("N2") + " %) - " + (len / 1048576.0).ToString("N1") + " MByte / " + (wdatRaw.Length / 1048576.0).ToString("N1") + " MByte");
// len += ProtoBuf.WriteVarInt(buf, len, ProtoBuf.UnsignedInt64(node.id - lastId));
// lastId = node.id;
// long nextPos = gpsPos.Int64Pos;
// len += ProtoBuf.WriteVarInt(buf, len, ProtoBuf.UnsignedInt64(nextPos - lastPos));
// lastPos = nextPos;
// len += ProtoBuf.WriteVarInt(buf, len, (uint)node.values.Length);
// foreach (var v in node.values)
// {
// len += ProtoBuf.WriteString(buf, len, v.Key);
// len += ProtoBuf.WriteString(buf, len, v.Value);
// }
// if (len > buf.Length - 65536)
// {
// block++;
// using (var wdat = File.Create("../tmp/node_block_" + block + "_" + lastId + ".dat"))
// {
// wdat.Write(buf, 0, len);
// len = 0;
// lastId = 0;
// lastPos = 0;
// }
// }
//}
}
if (rawLen > 0)
{
while (rawLen < rawBuf.Length)
{
rawBuf[rawLen++] = 0;
}
wdatRaw.Write(rawBuf, 0, rawBuf.Length);
}
if (len > 0)
{
block++;
using (var wdat = File.Create("../tmp/node_block_" + block + "_" + lastId + ".dat"))
{
wdat.Write(buf, 0, len);
}
}
//foreach (var way in pbf.ReadAllWays())
//{
// Console.WriteLine(way);
//}
//foreach (var relation in pbf.ReadAllRelations())
//{
// cc++;
// if ((byte)cc == 0)
// {
// Console.WriteLine(cc.ToString("N0") + " - " + relation);
// }
//}
}
}
static void WriteNodeBlock(Dictionary <string, int> keyDict, Dictionary <string, int> valDict, List <int> outputValues, List <DecodeNodeBlockValue> outputNodes, long lastId)
{
Func <Stream, byte[], int, int> write = (writeStream, buffer, len) =>
{
if (len > 0)
{
writeStream.Write(buffer, 0, len);
}
return(-len);
};
using (var wdat = File.Create("../tmp/node_sorted_" + lastId + ".dat"))
{
var buf = new byte[1048576];
int p = 8;
p += ProtoBuf.WriteVarInt(buf, p, (uint)keyDict.Count);
foreach (var k in keyDict.OrderBy(x => x.Value).Select(x => x.Key))
{
p += ProtoBuf.WriteString(buf, p, k);
if (p > 1000000)
{
p += write(wdat, buf, p);
}
}
p += ProtoBuf.WriteVarInt(buf, p, (uint)valDict.Count);
foreach (var k in valDict.OrderBy(x => x.Value).Select(x => x.Key))
{
p += ProtoBuf.WriteString(buf, p, k);
if (p > 1000000)
{
p += write(wdat, buf, p);
}
}
p += write(wdat, buf, p);
wdat.Position = 0;
wdat.Write(BitConverter.GetBytes((int)wdat.Length), 0, sizeof(int));
wdat.Position = wdat.Length;
p += ProtoBuf.WriteVarInt(buf, p, (uint)(outputValues.Count / 2));
foreach (var v in outputValues)
{
p += ProtoBuf.WriteVarInt(buf, p, (uint)v);
if (p > 1000000)
{
p += write(wdat, buf, p);
}
}
p += write(wdat, buf, p);
wdat.Position = 4;
wdat.Write(BitConverter.GetBytes((int)wdat.Length), 0, sizeof(int));
wdat.Position = wdat.Length;
p += ProtoBuf.WriteVarInt(buf, p, (uint)outputNodes.Count);
long pos = 0;
long id = 0;
int valuePos = 0;
foreach (var n in outputNodes)
{
p += ProtoBuf.WriteVarInt(buf, p, ProtoBuf.UnsignedInt64(n.pos - pos));
pos = n.pos;
p += ProtoBuf.WriteVarInt(buf, p, ProtoBuf.UnsignedInt64(n.id - id));
id = n.id;
p += ProtoBuf.WriteVarInt(buf, p, ProtoBuf.UnsignedInt64(n.valuePos - valuePos));
valuePos = n.valuePos;
p += ProtoBuf.WriteVarInt(buf, p, (uint)n.valueCount);
if (p > 1000000)
{
p += write(wdat, buf, p);
}
}
write(wdat, buf, p);
}
}