static void EncodeDelta(OsmWay[] ways)
{
for (int i = ways.Length - 1; i > 0; i--)
{
var ids = ways[i].nodeIds.ToArray();
for (int n = ids.Length - 1; n > 0; n--)
{
ids[n] = (long)ProtoBuf.UnsignedInt64(ids[n] - ids[n - 1]);
}
ways[i] = new OsmWay(ways[i].id - ways[i - 1].id, ways[i].values, ids);
}
}
static void ConverterTest_1_ExtractNodes()
{
Directory.CreateDirectory("../tmp");
using (var wdatRaw = File.Create("../tmp/node_rawfull.dat"))
using (var wdatRawIndex = File.Create("../tmp/node_rawfull_index.dat"))
using (var pbf = new OsmPbfReader(PbfPath))
{
long cc = 0;
var rawBuf = new byte[4096];
int rawLen = 0;
long rawId = 0;
long rawPos = 0;
var buf = new byte[2000 * 1048576];
int len = 0;
long lastId = 0;
long lastPos = 0;
int block = 0;
long totalPos = 0;
long totalSum = pbf.nodeIndex.Sum(x => x.nodeCount);
wdatRawIndex.Write(BitConverter.GetBytes(0L), 0, sizeof(long));
wdatRawIndex.Write(BitConverter.GetBytes(0L), 0, sizeof(long));
foreach (var node in pbf.ReadAllNodes())
{
totalPos++;
var gpsPos = new GpsPos(node);
rawLen += ProtoBuf.WriteVarInt(rawBuf, rawLen, ProtoBuf.UnsignedInt64(node.id - rawId));
rawId = node.id;
long nextFullPos = gpsPos.Int64Pos;
rawLen += ProtoBuf.WriteVarInt(rawBuf, rawLen, ProtoBuf.UnsignedInt64(nextFullPos - rawPos));
rawPos = nextFullPos;
if (rawLen > rawBuf.Length - 18)
{
while (rawLen < rawBuf.Length)
{
rawBuf[rawLen++] = 0;
}
wdatRaw.Write(rawBuf, 0, rawBuf.Length);
wdatRawIndex.Write(BitConverter.GetBytes(rawId + 1), 0, sizeof(long));
wdatRawIndex.Write(BitConverter.GetBytes(wdatRaw.Length), 0, sizeof(long));
rawLen = 0;
rawId = 0;
rawPos = 0;
}
//todo
//if (node.values.Length > 0)
//{
// cc++;
// if ((ushort)cc == 0) Console.WriteLine(cc.ToString("N0") + " (" + (100.0 / totalSum * totalPos).ToString("N2") + " %) - " + (len / 1048576.0).ToString("N1") + " MByte / " + (wdatRaw.Length / 1048576.0).ToString("N1") + " MByte");
// len += ProtoBuf.WriteVarInt(buf, len, ProtoBuf.UnsignedInt64(node.id - lastId));
// lastId = node.id;
// long nextPos = gpsPos.Int64Pos;
// len += ProtoBuf.WriteVarInt(buf, len, ProtoBuf.UnsignedInt64(nextPos - lastPos));
// lastPos = nextPos;
// len += ProtoBuf.WriteVarInt(buf, len, (uint)node.values.Length);
// foreach (var v in node.values)
// {
// len += ProtoBuf.WriteString(buf, len, v.Key);
// len += ProtoBuf.WriteString(buf, len, v.Value);
// }
// if (len > buf.Length - 65536)
// {
// block++;
// using (var wdat = File.Create("../tmp/node_block_" + block + "_" + lastId + ".dat"))
// {
// wdat.Write(buf, 0, len);
// len = 0;
// lastId = 0;
// lastPos = 0;
// }
// }
//}
}
if (rawLen > 0)
{
while (rawLen < rawBuf.Length)
{
rawBuf[rawLen++] = 0;
}
wdatRaw.Write(rawBuf, 0, rawBuf.Length);
}
if (len > 0)
{
block++;
using (var wdat = File.Create("../tmp/node_block_" + block + "_" + lastId + ".dat"))
{
wdat.Write(buf, 0, len);
}
}
//foreach (var way in pbf.ReadAllWays())
//{
// Console.WriteLine(way);
//}
//foreach (var relation in pbf.ReadAllRelations())
//{
// cc++;
// if ((byte)cc == 0)
// {
// Console.WriteLine(cc.ToString("N0") + " - " + relation);
// }
//}
}
}
static void WriteNodeBlock(Dictionary <string, int> keyDict, Dictionary <string, int> valDict, List <int> outputValues, List <DecodeNodeBlockValue> outputNodes, long lastId)
{
Func <Stream, byte[], int, int> write = (writeStream, buffer, len) =>
{
if (len > 0)
{
writeStream.Write(buffer, 0, len);
}
return(-len);
};
using (var wdat = File.Create("../tmp/node_sorted_" + lastId + ".dat"))
{
var buf = new byte[1048576];
int p = 8;
p += ProtoBuf.WriteVarInt(buf, p, (uint)keyDict.Count);
foreach (var k in keyDict.OrderBy(x => x.Value).Select(x => x.Key))
{
p += ProtoBuf.WriteString(buf, p, k);
if (p > 1000000)
{
p += write(wdat, buf, p);
}
}
p += ProtoBuf.WriteVarInt(buf, p, (uint)valDict.Count);
foreach (var k in valDict.OrderBy(x => x.Value).Select(x => x.Key))
{
p += ProtoBuf.WriteString(buf, p, k);
if (p > 1000000)
{
p += write(wdat, buf, p);
}
}
p += write(wdat, buf, p);
wdat.Position = 0;
wdat.Write(BitConverter.GetBytes((int)wdat.Length), 0, sizeof(int));
wdat.Position = wdat.Length;
p += ProtoBuf.WriteVarInt(buf, p, (uint)(outputValues.Count / 2));
foreach (var v in outputValues)
{
p += ProtoBuf.WriteVarInt(buf, p, (uint)v);
if (p > 1000000)
{
p += write(wdat, buf, p);
}
}
p += write(wdat, buf, p);
wdat.Position = 4;
wdat.Write(BitConverter.GetBytes((int)wdat.Length), 0, sizeof(int));
wdat.Position = wdat.Length;
p += ProtoBuf.WriteVarInt(buf, p, (uint)outputNodes.Count);
long pos = 0;
long id = 0;
int valuePos = 0;
foreach (var n in outputNodes)
{
p += ProtoBuf.WriteVarInt(buf, p, ProtoBuf.UnsignedInt64(n.pos - pos));
pos = n.pos;
p += ProtoBuf.WriteVarInt(buf, p, ProtoBuf.UnsignedInt64(n.id - id));
id = n.id;
p += ProtoBuf.WriteVarInt(buf, p, ProtoBuf.UnsignedInt64(n.valuePos - valuePos));
valuePos = n.valuePos;
p += ProtoBuf.WriteVarInt(buf, p, (uint)n.valueCount);
if (p > 1000000)
{
p += write(wdat, buf, p);
}
}
write(wdat, buf, p);
}
}