/// <summary> /// Does the v2 serialization. /// </summary> /// <param name="stream"></param> /// <param name="graph"></param> /// <returns></returns> protected override void DoSerialize(LimitedStream stream, DynamicGraphRouterDataSource <CHEdgeData> graph) { // sort the graph. IDynamicGraph <CHEdgeData> sortedGraph = CHEdgeDataDataSourceSerializer.SortGraph(graph); // create the regions. SortedDictionary <ulong, List <uint> > regions = new SortedDictionary <ulong, List <uint> >(); for (uint newVertexId = 1; newVertexId < sortedGraph.VertexCount + 1; newVertexId++) { // add to the CHRegions. float latitude, longitude; sortedGraph.GetVertex(newVertexId, out latitude, out longitude); Tile tile = Tile.CreateAroundLocation(new GeoCoordinate( latitude, longitude), RegionZoom); List <uint> regionVertices; if (!regions.TryGetValue(tile.Id, out regionVertices)) { regionVertices = new List <uint>(); regions.Add(tile.Id, regionVertices); } regionVertices.Add(newVertexId); } // serialize the sorted graph. // [START_OF_BLOCKS][[SIZE_OF_REGION_INDEX][REGION_INDEX][REGIONS]][[SIZE_OF_BLOCK_INDEX][BLOCK_INDEX][BLOCKS]] // STRART_OF_BLOCKS: 4bytes // SIZE_OF_REGION_INDEX: 4bytes // REGION_INDEX: see SIZE_OF_REGION_INDEX // REGIONS: see START_OF_BLOCKS - 4bytes // SIZE_OF_BLOCK_INDEX: 4bytes // BLOCK_INDEX: see SIZE_OF_BLOCK_INDEX. // BLOCKS: from START_OF_BLOCKS + 4bytes + SIZE_OF_BLOCKS_INDEX until END. // serialize regions and build their index. CHVertexRegionIndex chRegionIndex = new CHVertexRegionIndex(); chRegionIndex.LocationIndex = new int[regions.Count]; chRegionIndex.RegionIds = new ulong[regions.Count]; var memoryStream = new MemoryStream(); int regionIdx = 0; foreach (KeyValuePair <ulong, List <uint> > region in regions) { // serialize. CHVertexRegion chRegion = new CHVertexRegion(); chRegion.Vertices = region.Value.ToArray(); _runtimeTypeModel.Serialize(memoryStream, chRegion); // set index. chRegionIndex.LocationIndex[regionIdx] = (int)memoryStream.Position; chRegionIndex.RegionIds[regionIdx] = region.Key; regionIdx++; } stream.Seek(8, SeekOrigin.Begin); // move to beginning of [REGION_INDEX] _runtimeTypeModel.Serialize(stream, chRegionIndex); // write region index. int sizeRegionIndex = (int)(stream.Position - 8); // now at beginning of [REGIONS] memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.WriteTo(stream); // write regions. memoryStream.Dispose(); int startOfBlocks = (int)stream.Position; // now at beginning of [SIZE_OF_BLOCK_INDEX] stream.Seek(0, SeekOrigin.Begin); // move to beginning stream.Write(BitConverter.GetBytes(startOfBlocks), 0, 4); // write start position of blocks. Now at [SIZE_OF_REGION_INDEX] stream.Write(BitConverter.GetBytes(sizeRegionIndex), 0, 4); // write size of blocks index. // serialize the blocks and build their index. memoryStream = new MemoryStream(); List <int> blockLocations = new List <int>(); uint vertexId = 1; while (vertexId < sortedGraph.VertexCount) { uint blockId = vertexId; List <CHArc> blockArcs = new List <CHArc>(); List <CHVertex> blockVertices = new List <CHVertex>(); while (vertexId < blockId + BlockVertexSize && vertexId < sortedGraph.VertexCount + 1) { // create this block. CHVertex chVertex = new CHVertex(); float latitude, longitude; sortedGraph.GetVertex(vertexId, out latitude, out longitude); chVertex.Latitude = latitude; chVertex.Longitude = longitude; chVertex.ArcIndex = (ushort)(blockArcs.Count); foreach (KeyValuePair <uint, CHEdgeData> sortedArc in sortedGraph.GetArcs(vertexId)) { CHArc chArc = new CHArc(); chArc.TargetId = sortedArc.Key; chArc.ShortcutId = sortedArc.Value.ContractedVertexId; chArc.Weight = sortedArc.Value.Weight; chArc.Direction = sortedArc.Value.Direction; blockArcs.Add(chArc); } chVertex.ArcCount = (ushort)(blockArcs.Count - chVertex.ArcIndex); blockVertices.Add(chVertex); vertexId++; // move to the next vertex. } // create block. CHBlock block = new CHBlock(); block.Arcs = blockArcs.ToArray(); block.Vertices = blockVertices.ToArray(); // TODO: get rid of the list and create an array to begin with. // write blocks. _runtimeTypeModel.Serialize(memoryStream, block); blockLocations.Add((int)memoryStream.Position); } CHBlockIndex blockIndex = new CHBlockIndex(); blockIndex.LocationIndex = blockLocations.ToArray(); stream.Seek(startOfBlocks + 4, SeekOrigin.Begin); // move to beginning of [BLOCK_INDEX] _runtimeTypeModel.Serialize(stream, blockIndex); // write region index. int sizeBlockIndex = (int)(stream.Position - (startOfBlocks + 4)); // now at beginning of [BLOCKS] memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.WriteTo(stream); // write blocks. memoryStream.Dispose(); stream.Seek(startOfBlocks, SeekOrigin.Begin); // move to [SIZE_OF_BLOCK_INDEX] stream.Write(BitConverter.GetBytes(sizeBlockIndex), 0, 4); // write start position of blocks. Now at [SIZE_OF_REGION_INDEX] stream.Flush(); }
/// <summary> /// Does the v1 serialization. /// </summary> /// <param name="stream"></param> /// <param name="graph"></param> /// <returns></returns> protected override void DoSerialize(LimitedStream stream, DynamicGraphRouterDataSource <LiveEdge> graph) { // create an index per tile. var dataPerTile = new Dictionary <Tile, UnserializedTileData>(); for (uint vertex = 1; vertex < graph.VertexCount + 1; vertex++) { // loop over all vertices and serialize all into the correct tile. float latitude, longitude; if (graph.GetVertex(vertex, out latitude, out longitude)) { // the vertex was found. // build the correct tile. Tile tile = Tile.CreateAroundLocation(new GeoCoordinate(latitude, longitude), Zoom); UnserializedTileData serializableGraphTile; if (!dataPerTile.TryGetValue(tile, out serializableGraphTile)) { // create the new tile. serializableGraphTile = new UnserializedTileData(); serializableGraphTile.Ids = new List <uint>(); serializableGraphTile.Latitude = new List <ushort>(); serializableGraphTile.Longitude = new List <ushort>(); serializableGraphTile.StringTable = new Dictionary <string, int>(); serializableGraphTile.Arcs = new List <SerializableGraphArcs>(); dataPerTile.Add(tile, serializableGraphTile); } // create short latitude/longitude. serializableGraphTile.Ids.Add(vertex); serializableGraphTile.Latitude.Add((ushort)(((tile.TopLeft.Latitude - latitude) / tile.Box.DeltaLat) * ushort.MaxValue)); serializableGraphTile.Longitude.Add((ushort)(((longitude - tile.TopLeft.Longitude) / tile.Box.DeltaLon) * ushort.MaxValue)); // get the arcs. KeyValuePair <uint, LiveEdge>[] arcs = graph.GetArcs(vertex); // serialize the arcs. if (arcs != null && arcs.Length > 0) { var serializableGraphArcs = new SerializableGraphArcs(); serializableGraphArcs.DestinationId = new uint[arcs.Length]; serializableGraphArcs.Forward = new bool[arcs.Length]; serializableGraphArcs.TileX = new int[arcs.Length]; serializableGraphArcs.TileY = new int[arcs.Length]; serializableGraphArcs.Tags = new SerializableTags[arcs.Length]; for (int idx = 0; idx < arcs.Length; idx++) { KeyValuePair <uint, LiveEdge> arc = arcs[idx]; // get destination tile. if (graph.GetVertex(arc.Key, out latitude, out longitude)) { // the destionation was found. Tile destinationTile = Tile.CreateAroundLocation( new GeoCoordinate(latitude, longitude), Zoom); serializableGraphArcs.DestinationId[idx] = arc.Key; serializableGraphArcs.TileX[idx] = destinationTile.X; serializableGraphArcs.TileY[idx] = destinationTile.Y; serializableGraphArcs.Forward[idx] = arc.Value.Forward; // get the tags. TagsCollectionBase tagsCollection = graph.TagsIndex.Get(arc.Value.Tags); if (tagsCollection != null) { serializableGraphArcs.Tags[idx] = new SerializableTags(); serializableGraphArcs.Tags[idx].Keys = new int[tagsCollection.Count]; serializableGraphArcs.Tags[idx].Values = new int[tagsCollection.Count]; int tagsIndex = 0; foreach (var tag in tagsCollection) { int key; if (!serializableGraphTile.StringTable.TryGetValue( tag.Key, out key)) { // string not yet in string table. key = serializableGraphTile.StringTable.Count; serializableGraphTile.StringTable.Add(tag.Key, key); } int value; if (!serializableGraphTile.StringTable.TryGetValue( tag.Value, out value)) { // string not yet in string table. value = serializableGraphTile.StringTable.Count; serializableGraphTile.StringTable.Add(tag.Value, value); } serializableGraphArcs.Tags[idx].Keys[tagsIndex] = key; serializableGraphArcs.Tags[idx].Values[tagsIndex] = value; tagsIndex++; } } } } serializableGraphTile.Arcs.Add(serializableGraphArcs); } } } // LAYOUT OF V2: {HEADER}{compressionflag(1byte)}{#tiles(4byte)}{tilesMetaEnd(8byte)}{tiles-meta-data-xxxxxxx}{tiles-data} // {HEADER} : already written before this method. // {#tiles(4byte)} : the number of tiles in this file (calculate the offset of the {tiles-data} // section using (TileMetaSize * dataPerTile.Count + 4 + 8) // {tilesMetaEnd(8byte)} : the end of the meta tiles. // {tiles-meta-data-xxxxxxx} : the serialized tile metadata. // {tiles-data} : the actual tile data. // calculate the space needed for the tile offset. const long tileMetaOffset = 1 + 4 + 8; long tileOffset = TileMetaSize * dataPerTile.Count + tileMetaOffset; // all tile metadata + a tile count + tags offset. // build the tile metadata while writing the tile data. stream.Seek(tileOffset, SeekOrigin.Begin); var metas = new SerializableGraphTileMetas(); metas.Length = new int[dataPerTile.Count]; metas.Offset = new long[dataPerTile.Count]; metas.TileX = new int[dataPerTile.Count]; metas.TileY = new int[dataPerTile.Count]; int metasIndex = 0; foreach (var unserializedTileData in dataPerTile) { // create the tile meta. metas.TileX[metasIndex] = unserializedTileData.Key.X; metas.TileY[metasIndex] = unserializedTileData.Key.Y; metas.Offset[metasIndex] = stream.Position; // create the tile. var serializableGraphTile = new SerializableGraphTile(); serializableGraphTile.Arcs = unserializedTileData.Value.Arcs.ToArray(); serializableGraphTile.Ids = unserializedTileData.Value.Ids.ToArray(); serializableGraphTile.Latitude = unserializedTileData.Value.Latitude.ToArray(); serializableGraphTile.Longitude = unserializedTileData.Value.Longitude.ToArray(); serializableGraphTile.StringTable = new string[unserializedTileData.Value.StringTable.Count]; foreach (var stringEntry in unserializedTileData.Value.StringTable) { serializableGraphTile.StringTable[stringEntry.Value] = stringEntry.Key; } // serialize the tile. if (!_compress) { // compresses the file. _runtimeTypeModel.Serialize(stream, serializableGraphTile); } else { // first compress the data, then write. var uncompressed = new MemoryStream(); _runtimeTypeModel.Serialize(uncompressed, serializableGraphTile); var uncompressedBuffer = uncompressed.ToArray(); byte[] compressed = GZipStream.CompressBuffer(uncompressedBuffer); stream.Write(compressed, 0, compressed.Length); } // calculate the length of the data that was just serialized. metas.Length[metasIndex] = (int)(stream.Position - metas.Offset[metasIndex]); metasIndex++; } // serialize all tile meta data. stream.Seek(tileMetaOffset, SeekOrigin.Begin); _runtimeTypeModel.Serialize(stream, metas); long tileMetaEnd = stream.Position; // save the meta and. // save all the offsets. stream.Seek(0, SeekOrigin.Begin); byte[] compressionFlag = new[] { (byte)(_compress ? 1 : 0) }; stream.Write(compressionFlag, 0, 1); byte[] tileCountBytes = BitConverter.GetBytes(metas.TileX.Length); stream.Write(tileCountBytes, 0, tileCountBytes.Length); // 4 bytes byte[] tileMetaEndBytes = BitConverter.GetBytes(tileMetaEnd); stream.Write(tileMetaEndBytes, 0, tileMetaEndBytes.Length); // 8 bytes stream.Flush(); }
/// <summary> /// Does the v2 serialization. /// </summary> /// <param name="stream"></param> /// <param name="graph"></param> /// <returns></returns> protected override void DoSerialize(LimitedStream stream, DynamicGraphRouterDataSource <CHEdgeData> graph) { // sort the graph. var sortedGraph = graph; // create the regions. var regions = new SortedDictionary <ulong, List <uint> >(); for (uint newVertexId = 1; newVertexId < sortedGraph.VertexCount + 1; newVertexId++) { // add to the CHRegions. float latitude, longitude; sortedGraph.GetVertex(newVertexId, out latitude, out longitude); var tile = Tile.CreateAroundLocation(new GeoCoordinate( latitude, longitude), _regionZoom); List <uint> regionVertices; if (!regions.TryGetValue(tile.Id, out regionVertices)) { regionVertices = new List <uint>(); regions.Add(tile.Id, regionVertices); } regionVertices.Add(newVertexId); } // serialize the sorted graph. // [START_OF_BLOCKS][START_OF_SHAPES][START_OF_REVERSE][START_OF_TAGS][[SIZE_OF_REGION_INDEX][REGION_INDEX][REGIONS]] // [[SIZE_OF_BLOCK_INDEX][BLOCK_INDEX][BLOCKS]] // [[SIZE_OF_SHAPE_INDEX][SHAPE_INDEX][SHAPES]] // [[SIZE_OF_REVERSE_INDEX][REVERSE_INDEX][REVERSE]] // [TAGS] // START_OF_BLOCKS: 4bytes // START_OF_SHAPES: 4bytes // START_OF_STAGS: 4bytes // START_OF_REVERSE 4bytes // SIZE_OF_REGION_INDEX: 4bytes // REGION_INDEX: see SIZE_OF_REGION_INDEX // REGIONS: see START_OF_BLOCKS - 4bytes // SIZE_OF_BLOCK_INDEX: 4bytes // BLOCK_INDEX: see SIZE_OF_BLOCK_INDEX. // BLOCKS: from START_OF_BLOCKS + 4bytes + SIZE_OF_BLOCKS_INDEX until END. // serialize regions and build their index. var chRegionIndex = new CHVertexRegionIndex(); chRegionIndex.LocationIndex = new int[regions.Count]; chRegionIndex.RegionIds = new ulong[regions.Count]; var memoryStream = new MemoryStream(); int regionIdx = 0; foreach (var region in regions) { // serialize. var chRegion = new CHVertexRegion(); chRegion.Vertices = region.Value.ToArray(); _runtimeTypeModel.Serialize(memoryStream, chRegion); // set index. chRegionIndex.LocationIndex[regionIdx] = (int)memoryStream.Position; chRegionIndex.RegionIds[regionIdx] = region.Key; regionIdx++; } stream.Seek(INDEX_SIZE, SeekOrigin.Begin); // move to beginning of [REGION_INDEX] _runtimeTypeModel.Serialize(stream, chRegionIndex); // write region index. var sizeRegionIndex = (int)(stream.Position - INDEX_SIZE); // now at beginning of [REGIONS] memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.WriteTo(stream); // write regions. memoryStream.Dispose(); var startOfBlocks = (int)stream.Position; // now at beginning of [SIZE_OF_BLOCK_INDEX] // serialize the blocks and build their index. memoryStream = new MemoryStream(); var blocks = new List <CHBlock>(); var blockLocations = new List <int>(); var blockShapeLocations = new List <int>(); var blockReverseLocations = new List <int>(); var blockShapes = new List <CHBlockCoordinates>(); var reverseNeighbours = new Dictionary <uint, List <uint> >(); uint vertexId = 1; while (vertexId < sortedGraph.VertexCount + 1) { uint blockId = vertexId; var blockArcs = new List <CHArc>(); var blockArcCoordinates = new List <CHArcCoordinates>(); var blockVertices = new List <CHVertex>(); while (vertexId < blockId + _blockVertexSize && vertexId < sortedGraph.VertexCount + 1) { // create this block. var chVertex = new CHVertex(); float latitude, longitude; sortedGraph.GetVertex(vertexId, out latitude, out longitude); chVertex.Latitude = latitude; chVertex.Longitude = longitude; chVertex.ArcIndex = (ushort)(blockArcs.Count); List <uint> reverse = null; foreach (var sortedArc in sortedGraph.GetEdges(vertexId)) { var chArc = new CHArc(); chArc.TargetId = sortedArc.Neighbour; chArc.Meta = sortedArc.EdgeData.Meta; chArc.Value = sortedArc.EdgeData.Value; chArc.Weight = sortedArc.EdgeData.Weight; blockArcs.Add(chArc); var chArcCoordinates = new CHArcCoordinates(); chArcCoordinates.Coordinates = sortedArc.Intermediates.ToSimpleArray(); blockArcCoordinates.Add(chArcCoordinates); if (sortedArc.EdgeData.RepresentsNeighbourRelations) { if (!reverseNeighbours.TryGetValue(sortedArc.Neighbour, out reverse)) { reverse = new List <uint>(); reverseNeighbours[sortedArc.Neighbour] = reverse; } reverse.Add(vertexId); } } chVertex.ArcCount = (ushort)(blockArcs.Count - chVertex.ArcIndex); blockVertices.Add(chVertex); vertexId++; // move to the next vertex. } // create block. var block = new CHBlock(); block.Arcs = blockArcs.ToArray(); block.Vertices = blockVertices.ToArray(); // TODO: get rid of the list and create an array to begin with. blocks.Add(block); var blockCoordinates = new CHBlockCoordinates(); blockCoordinates.Arcs = blockArcCoordinates.ToArray(); // write blocks. var blockStream = new MemoryStream(); _runtimeTypeModel.Serialize(blockStream, block); var compressed = GZipStream.CompressBuffer(blockStream.ToArray()); blockStream.Dispose(); memoryStream.Write(compressed, 0, compressed.Length); blockLocations.Add((int)memoryStream.Position); blockShapes.Add(blockCoordinates); } var blockIndex = new CHBlockIndex(); blockIndex.BlockLocationIndex = blockLocations.ToArray(); stream.Seek(startOfBlocks + 4, SeekOrigin.Begin); // move to beginning of [BLOCK_INDEX] _runtimeTypeModel.Serialize(stream, blockIndex); // write block index. int sizeBlockIndex = (int)(stream.Position - (startOfBlocks + 4)); // now at beginning of [BLOCKS] memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.WriteTo(stream); // write blocks. memoryStream.Dispose(); // write shapes and index. memoryStream = new MemoryStream(); var startOfShapes = stream.Position; foreach (var blockCoordinates in blockShapes) { // write shape blocks. var blockShapeStream = new MemoryStream(); _runtimeTypeModel.Serialize(blockShapeStream, blockCoordinates); var compressed = GZipStream.CompressBuffer(blockShapeStream.ToArray()); blockShapeStream.Dispose(); memoryStream.Write(compressed, 0, compressed.Length); blockShapeLocations.Add((int)memoryStream.Position); } blockIndex = new CHBlockIndex(); blockIndex.BlockLocationIndex = blockShapeLocations.ToArray(); stream.Seek(startOfShapes + 4, SeekOrigin.Begin); // move to beginning of [SHAPE_INDEX] _runtimeTypeModel.Serialize(stream, blockIndex); // write shape index. int sizeShapeIndex = (int)(stream.Position - (startOfShapes + 4)); // now at beginning of [SHAPE] memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.WriteTo(stream); // write shapes. memoryStream.Dispose(); // build reverse blocks. var blockReverses = new List <CHBlockReverse>(); List <uint> storedReverses = null; uint currentVertexId = 0; for (int i = 0; i < blocks.Count; i++) { var blockReverse = new CHBlockReverse(); blockReverse.Vertices = new CHArcReverse[blocks[i].Vertices.Length]; for (int j = 0; j < blocks[i].Vertices.Length; j++) { currentVertexId++; var blockReverseArc = new CHArcReverse(); blockReverseArc.Neighbours = new uint[0]; if (reverseNeighbours.TryGetValue(currentVertexId, out storedReverses)) { blockReverseArc.Neighbours = storedReverses.ToArray(); } blockReverse.Vertices[j] = blockReverseArc; } blockReverses.Add(blockReverse); } // write reverse and index. memoryStream = new MemoryStream(); var startOfReverses = stream.Position; foreach (var blockReverse in blockReverses) { // write reverse blocks. var blockReverseStream = new MemoryStream(); _runtimeTypeModel.Serialize(blockReverseStream, blockReverse); var compressed = GZipStream.CompressBuffer(blockReverseStream.ToArray()); blockReverseStream.Dispose(); memoryStream.Write(compressed, 0, compressed.Length); blockReverseLocations.Add((int)memoryStream.Position); } blockIndex = new CHBlockIndex(); blockIndex.BlockLocationIndex = blockReverseLocations.ToArray(); stream.Seek(startOfReverses + 4, SeekOrigin.Begin); // move to beginning of [REVERSE_INDEX] _runtimeTypeModel.Serialize(stream, blockIndex); // write reverse index. int sizeReverseIndex = (int)(stream.Position - (startOfReverses + 4)); // now at beginning of [REVERSE] memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.WriteTo(stream); // write reverses. memoryStream.Dispose(); // write tags after reverses. int tagsPosition = (int)stream.Position; TagIndexSerializer.SerializeBlocks(stream, graph.TagsIndex, 10); stream.Seek(startOfBlocks, SeekOrigin.Begin); // move to [SIZE_OF_BLOCK_INDEX] stream.Write(BitConverter.GetBytes(sizeBlockIndex), 0, 4); // write start position of blocks. stream.Seek(startOfShapes, SeekOrigin.Begin); // move to [SIZE_OF_BLOCK_INDEX] stream.Write(BitConverter.GetBytes(sizeShapeIndex), 0, 4); // write start position of blocks. stream.Seek(startOfReverses, SeekOrigin.Begin); // move to [SIZE_OF_REVERSE_INDEX] stream.Write(BitConverter.GetBytes(sizeReverseIndex), 0, 4); // write start position of blocks. // write index. stream.Seek(0, SeekOrigin.Begin); // move to beginning stream.Write(BitConverter.GetBytes(startOfBlocks), 0, 4); // write start position of blocks. stream.Write(BitConverter.GetBytes(startOfShapes), 0, 4); // write start position of shapes. stream.Write(BitConverter.GetBytes(startOfReverses), 0, 4); // write start position of shapes. stream.Write(BitConverter.GetBytes(tagsPosition), 0, 4); stream.Write(BitConverter.GetBytes(sizeRegionIndex), 0, 4); // write size of region index. stream.Flush(); }