void WriteIndex(MemberType type, long id, long index) { switch (type) { case MemberType.Relation: relationsIndex.Add(id, index); break; case MemberType.Way: waysIndex.Add(id, index); break; case MemberType.Node: nodesIndex.Add(id, index); break; default: throw new Exception("unknown type?"); } byte[] miniBuf = new byte[64]; int p = 0; miniBuf[p] = (byte)type; p++; p += ProtoBuf.WriteVarInt(miniBuf, p, (ulong)id); p += ProtoBuf.WriteVarInt(miniBuf, p, (ulong)index); using (var indexFile = File.OpenWrite(CacheIndexFile)) { indexFile.Position = indexFile.Length; indexFile.Write(miniBuf, 0, p); } }
long WriteRelationCache(OsmRelation osmRelation, OsmWay[] osmWays, OsmNode[] osmNodes) { long relationIndex = cacheData.Length; int p = sizeof(int); // cacheblock-size var buf = cacheBuffer; p += osmRelation.WriteBinary(buf, p); p += ProtoBuf.WriteVarInt(buf, p, (ulong)osmWays.Length); var tmpWays = osmWays.ToArray(); EncodeDelta(tmpWays); foreach (var way in tmpWays) { p += way.WriteBinary(buf, p); } p += ProtoBuf.WriteVarInt(buf, p, (ulong)osmNodes.Length); var tmpNodes = osmNodes.ToArray(); EncodeDelta(tmpNodes); foreach (var node in tmpNodes) { p += node.WriteBinary(buf, p); } buf[0] = (byte)p; buf[1] = (byte)(p >> 8); buf[2] = (byte)(p >> 16); buf[3] = (byte)(p >> 24); cacheData.Position = relationIndex; cacheData.Write(buf, 0, p); return(relationIndex); }
public int WriteBinary(byte[] buf, int ofs) { int p = 0; p += ProtoBuf.WriteVarInt(buf, ofs + p, (ulong)id); buf[ofs + p] = (byte)type; p++; p += ProtoBuf.WriteString(buf, ofs + p, role); return(p); }
static int WriteNodesPath(byte[] buf, int ofs, OsmNode[] nodes) { int p = ProtoBuf.WriteVarInt(buf, ofs, (uint)nodes.Length); foreach (var node in nodes) { p += node.WriteBinary(buf, ofs + p); } return(p); }
public int WriteBinary(byte[] buf, int ofs) { int p = 0; p += ProtoBuf.WriteVarInt(buf, ofs + p, (ulong)id); p += ProtoBuf.WriteVarInt(buf, ofs + p, ProtoBuf.UnsignedInt32(latCode)); p += ProtoBuf.WriteVarInt(buf, ofs + p, ProtoBuf.UnsignedInt32(lonCode)); p += ProtoBuf.WriteVarInt(buf, ofs + p, (uint)values.Length); foreach (var val in values) { p += ProtoBuf.WriteString(buf, ofs + p, val.Key); p += ProtoBuf.WriteString(buf, ofs + p, val.Value); } return(p); }
public int WriteBinary(byte[] buf, int ofs) { int p = 0; p += ProtoBuf.WriteVarInt(buf, ofs + p, (ulong)id); p += ProtoBuf.WriteVarInt(buf, ofs + p, (uint)values.Length); foreach (var val in values) { p += ProtoBuf.WriteString(buf, ofs + p, val.Key); p += ProtoBuf.WriteString(buf, ofs + p, val.Value); } p += ProtoBuf.WriteVarInt(buf, ofs + p, (uint)members.Length); foreach (var member in members) { p += member.WriteBinary(buf, ofs + p); } return(p); }
static void ConverterTest_1_ExtractNodes() { Directory.CreateDirectory("../tmp"); using (var wdatRaw = File.Create("../tmp/node_rawfull.dat")) using (var wdatRawIndex = File.Create("../tmp/node_rawfull_index.dat")) using (var pbf = new OsmPbfReader(PbfPath)) { long cc = 0; var rawBuf = new byte[4096]; int rawLen = 0; long rawId = 0; long rawPos = 0; var buf = new byte[2000 * 1048576]; int len = 0; long lastId = 0; long lastPos = 0; int block = 0; long totalPos = 0; long totalSum = pbf.nodeIndex.Sum(x => x.nodeCount); wdatRawIndex.Write(BitConverter.GetBytes(0L), 0, sizeof(long)); wdatRawIndex.Write(BitConverter.GetBytes(0L), 0, sizeof(long)); foreach (var node in pbf.ReadAllNodes()) { totalPos++; var gpsPos = new GpsPos(node); rawLen += ProtoBuf.WriteVarInt(rawBuf, rawLen, ProtoBuf.UnsignedInt64(node.id - rawId)); rawId = node.id; long nextFullPos = gpsPos.Int64Pos; rawLen += ProtoBuf.WriteVarInt(rawBuf, rawLen, ProtoBuf.UnsignedInt64(nextFullPos - rawPos)); rawPos = nextFullPos; if (rawLen > rawBuf.Length - 18) { while (rawLen < rawBuf.Length) { rawBuf[rawLen++] = 0; } wdatRaw.Write(rawBuf, 0, rawBuf.Length); wdatRawIndex.Write(BitConverter.GetBytes(rawId + 1), 0, sizeof(long)); wdatRawIndex.Write(BitConverter.GetBytes(wdatRaw.Length), 0, sizeof(long)); rawLen = 0; rawId = 0; rawPos = 0; } //todo //if (node.values.Length > 0) //{ // cc++; // if ((ushort)cc == 0) Console.WriteLine(cc.ToString("N0") + " (" + (100.0 / totalSum * totalPos).ToString("N2") + " %) - " + (len / 1048576.0).ToString("N1") + " MByte / " + (wdatRaw.Length / 1048576.0).ToString("N1") + " MByte"); // len += ProtoBuf.WriteVarInt(buf, len, ProtoBuf.UnsignedInt64(node.id - lastId)); // lastId = node.id; // long nextPos = gpsPos.Int64Pos; // len += ProtoBuf.WriteVarInt(buf, len, ProtoBuf.UnsignedInt64(nextPos - lastPos)); // lastPos = nextPos; // len += ProtoBuf.WriteVarInt(buf, len, (uint)node.values.Length); // foreach (var v in node.values) // { // len += ProtoBuf.WriteString(buf, len, v.Key); // len += ProtoBuf.WriteString(buf, len, v.Value); // } // if (len > buf.Length - 65536) // { // block++; // using (var wdat = File.Create("../tmp/node_block_" + block + "_" + lastId + ".dat")) // { // wdat.Write(buf, 0, len); // len = 0; // lastId = 0; // lastPos = 0; // } // } //} } if (rawLen > 0) { while (rawLen < rawBuf.Length) { rawBuf[rawLen++] = 0; } wdatRaw.Write(rawBuf, 0, rawBuf.Length); } if (len > 0) { block++; using (var wdat = File.Create("../tmp/node_block_" + block + "_" + lastId + ".dat")) { wdat.Write(buf, 0, len); } } //foreach (var way in pbf.ReadAllWays()) //{ // Console.WriteLine(way); //} //foreach (var relation in pbf.ReadAllRelations()) //{ // cc++; // if ((byte)cc == 0) // { // Console.WriteLine(cc.ToString("N0") + " - " + relation); // } //} } }
static void WriteNodeBlock(Dictionary <string, int> keyDict, Dictionary <string, int> valDict, List <int> outputValues, List <DecodeNodeBlockValue> outputNodes, long lastId) { Func <Stream, byte[], int, int> write = (writeStream, buffer, len) => { if (len > 0) { writeStream.Write(buffer, 0, len); } return(-len); }; using (var wdat = File.Create("../tmp/node_sorted_" + lastId + ".dat")) { var buf = new byte[1048576]; int p = 8; p += ProtoBuf.WriteVarInt(buf, p, (uint)keyDict.Count); foreach (var k in keyDict.OrderBy(x => x.Value).Select(x => x.Key)) { p += ProtoBuf.WriteString(buf, p, k); if (p > 1000000) { p += write(wdat, buf, p); } } p += ProtoBuf.WriteVarInt(buf, p, (uint)valDict.Count); foreach (var k in valDict.OrderBy(x => x.Value).Select(x => x.Key)) { p += ProtoBuf.WriteString(buf, p, k); if (p > 1000000) { p += write(wdat, buf, p); } } p += write(wdat, buf, p); wdat.Position = 0; wdat.Write(BitConverter.GetBytes((int)wdat.Length), 0, sizeof(int)); wdat.Position = wdat.Length; p += ProtoBuf.WriteVarInt(buf, p, (uint)(outputValues.Count / 2)); foreach (var v in outputValues) { p += ProtoBuf.WriteVarInt(buf, p, (uint)v); if (p > 1000000) { p += write(wdat, buf, p); } } p += write(wdat, buf, p); wdat.Position = 4; wdat.Write(BitConverter.GetBytes((int)wdat.Length), 0, sizeof(int)); wdat.Position = wdat.Length; p += ProtoBuf.WriteVarInt(buf, p, (uint)outputNodes.Count); long pos = 0; long id = 0; int valuePos = 0; foreach (var n in outputNodes) { p += ProtoBuf.WriteVarInt(buf, p, ProtoBuf.UnsignedInt64(n.pos - pos)); pos = n.pos; p += ProtoBuf.WriteVarInt(buf, p, ProtoBuf.UnsignedInt64(n.id - id)); id = n.id; p += ProtoBuf.WriteVarInt(buf, p, ProtoBuf.UnsignedInt64(n.valuePos - valuePos)); valuePos = n.valuePos; p += ProtoBuf.WriteVarInt(buf, p, (uint)n.valueCount); if (p > 1000000) { p += write(wdat, buf, p); } } write(wdat, buf, p); } }