/// <summary> /// Does the v1 deserialization. /// </summary> /// <param name="stream"></param> /// <param name="lazy"></param> /// <param name="vehicles"></param> /// <returns></returns> protected override IBasicRouterDataSource <TEdgeData> DoDeserialize( LimitedStream stream, bool lazy, IEnumerable <string> vehicles) { ITagsCollectionIndex tagsCollectionIndex = this.CreateTagsCollectionIndex(); DynamicGraphRouterDataSource <TEdgeData> graph = this.CreateGraph(tagsCollectionIndex); // deserialize vertices. var sizeBytes = new byte[8]; stream.Read(sizeBytes, 0, 8); var position = stream.Position; var size = BitConverter.ToInt64(sizeBytes, 0); this.DeserializeVertices(stream, size, graph); stream.Seek(position + size, System.IO.SeekOrigin.Begin); // deserialize edges. stream.Read(sizeBytes, 0, 8); position = stream.Position; size = BitConverter.ToInt64(sizeBytes, 0); this.DeserializeEdges(stream, size, graph); stream.Seek(position + size, System.IO.SeekOrigin.Begin); // deserialize tags. stream.Read(sizeBytes, 0, 8); position = stream.Position; size = BitConverter.ToInt64(sizeBytes, 0); this.DeserializeTags(stream, size, tagsCollectionIndex); stream.Seek(position + size, System.IO.SeekOrigin.Begin); return(graph); }
/// <summary> /// Does the v2 deserialization. /// </summary> /// <param name="stream"></param> /// <param name="lazy"></param> /// <param name="vehicles"></param> /// <returns></returns> protected override IBasicRouterDataSource <CHEdgeData> DoDeserialize( LimitedStream stream, bool lazy, IEnumerable <string> vehicles) { var intBytes = new byte[4]; stream.Read(intBytes, 0, 4); int startOfBlocks = BitConverter.ToInt32(intBytes, 0); stream.Read(intBytes, 0, 4); int startOfTags = BitConverter.ToInt32(intBytes, 0); stream.Read(intBytes, 0, 4); int sizeRegionIndex = BitConverter.ToInt32(intBytes, 0); // deserialize regions index. var chVertexRegionIndex = (CHVertexRegionIndex)_runtimeTypeModel.Deserialize( new CappedStream(stream, stream.Position, sizeRegionIndex), null, typeof(CHVertexRegionIndex)); // deserialize blocks index. stream.Seek(startOfBlocks, SeekOrigin.Begin); stream.Read(intBytes, 0, 4); int sizeBlockIndex = BitConverter.ToInt32(intBytes, 0); var chBlockIndex = (CHBlockIndex)_runtimeTypeModel.Deserialize( new CappedStream(stream, stream.Position, sizeBlockIndex), null, typeof(CHBlockIndex)); // deserialize tags. stream.Seek(startOfTags, SeekOrigin.Begin); ITagsCollectionIndexReadonly tagsIndex = TagIndexSerializer.DeserializeBlocks(stream); return(new v2.CHEdgeDataDataSource(stream, this, vehicles, sizeRegionIndex + 12, chVertexRegionIndex, _regionZoom, startOfBlocks + sizeBlockIndex + 4, chBlockIndex, (uint)_blockVertexSize, tagsIndex)); }
/// <summary> /// Serialize/deserialize index. /// </summary> /// <param name="index"></param> /// <param name="blockSize"></param> /// <param name="position"></param> /// <returns></returns> private ITagsCollectionIndexReadonly SerializeDeserializeBlockLimitedStream(ITagsCollectionIndex index, uint blockSize, int position) { MemoryStream memoryStream = new MemoryStream(); memoryStream.Seek(position, SeekOrigin.Begin); LimitedStream stream = new LimitedStream(memoryStream); stream.Seek(position, SeekOrigin.Begin); TagIndexSerializer.SerializeBlocks(stream, index, blockSize); stream.Seek(position, SeekOrigin.Begin); return(TagIndexSerializer.DeserializeBlocks(stream)); }
/// <summary> /// Does the v2 deserialization. /// </summary> /// <param name="stream"></param> /// <param name="lazy"></param> /// <returns></returns> protected override IBasicRouterDataSource <CHEdgeData> DoDeserialize( LimitedStream stream, bool lazy) { var intBytes = new byte[4]; stream.Read(intBytes, 0, 4); int startOfBlocks = BitConverter.ToInt32(intBytes, 0); // deserialize regions index. stream.Read(intBytes, 0, 4); int sizeRegionIndex = BitConverter.ToInt32(intBytes, 0); var chVertexRegionIndex = (CHVertexRegionIndex)_runtimeTypeModel.Deserialize( new CappedStream(stream, stream.Position, sizeRegionIndex), null, typeof(CHVertexRegionIndex)); // deserialize blocks index. stream.Seek(startOfBlocks, SeekOrigin.Begin); stream.Read(intBytes, 0, 4); int sizeBlockIndex = BitConverter.ToInt32(intBytes, 0); var chBlockIndex = (CHBlockIndex)_runtimeTypeModel.Deserialize( new CappedStream(stream, stream.Position, sizeBlockIndex), null, typeof(CHBlockIndex)); return(new CHEdgeDataDataSource(stream, this, sizeRegionIndex + 8, chVertexRegionIndex, RegionZoom, startOfBlocks + sizeBlockIndex + 4, chBlockIndex, BlockVertexSize)); }
/// <summary> /// Serialize/deserialize index. /// </summary> /// <param name="index"></param> /// <param name="position"></param> /// <returns></returns> private ITagsIndexReadonly SerializeDeserializeLimitedStream(ITagsIndex index, int position) { MemoryStream memoryStream = new MemoryStream(); memoryStream.Seek(position, SeekOrigin.Begin); LimitedStream stream = new LimitedStream(memoryStream); TagIndexSerializer.Serialize(stream, index); stream.Seek(0, SeekOrigin.Begin); return(TagIndexSerializer.Deserialize(stream)); }
public void Seek_Origin_Begin_LessLimit_Throw_IOException() { var source = CreateSourceStream(); const int stream_offset = 5; const int stream_length = 10; const long offset = -2; var limited = new LimitedStream(source, stream_offset, stream_length); var error = Assert.ThrowsException <IOException>(() => limited.Seek(offset, SeekOrigin.Begin)); Assert.That.Value(error.Message) .IsEqual("An attempt was made to move the position before the beginning of the stream."); Assert.That.Value(error.Data["offset"]).IsEqual(offset); }
/// <summary> /// Does the v1 serialization. /// </summary> /// <param name="stream"></param> /// <param name="graph"></param> /// <returns></returns> protected override void DoSerialize(LimitedStream stream, DynamicGraphRouterDataSource <TEdgeData> graph) { // LAYOUT: // [SIZE_OF_VERTICES(8bytes)][VERTICES][SIZE_OF_EDGES(8bytes)][EDGES][SIZE_OF_TAGS(8bytes][TAGS] // serialize coordinates. stream.Seek(8, System.IO.SeekOrigin.Current); long position = stream.Position; this.SerializeVertices(stream, graph); long size = stream.Position - position; stream.Seek(position - 8, System.IO.SeekOrigin.Begin); var sizeBytes = BitConverter.GetBytes(size); stream.Write(sizeBytes, 0, 8); stream.Seek(size, System.IO.SeekOrigin.Current); // serialize edges. stream.Seek(8, System.IO.SeekOrigin.Current); position = stream.Position; this.SerializeEdges(stream, graph); size = stream.Position - position; stream.Seek(position - 8, System.IO.SeekOrigin.Begin); sizeBytes = BitConverter.GetBytes(size); stream.Write(sizeBytes, 0, 8); stream.Seek(size, System.IO.SeekOrigin.Current); // serialize tags. stream.Seek(8, System.IO.SeekOrigin.Current); position = stream.Position; this.SerializeTags(stream, graph.TagsIndex); size = stream.Position - position; stream.Seek(position - 8, System.IO.SeekOrigin.Begin); sizeBytes = BitConverter.GetBytes(size); stream.Write(sizeBytes, 0, 8); stream.Seek(size, System.IO.SeekOrigin.Current); }
/// <summary> /// Does the v1 deserialization. /// </summary> /// <param name="stream"></param> /// <param name="lazy"></param> /// <param name="vehicles"></param> /// <returns></returns> protected override IBasicRouterDataSource <CHEdgeData> DoDeserialize( LimitedStream stream, bool lazy, IEnumerable <string> vehicles) { // serialize all tile meta data. stream.Seek(0, SeekOrigin.Begin); var compressionFlag = new byte[1]; stream.Read(compressionFlag, 0, 1); bool decompress = (compressionFlag[0] == (byte)1); var tileCountBytes = new byte[4]; stream.Read(tileCountBytes, 0, tileCountBytes.Length); // var tileCount = BitConverter.ToInt32(tileCountBytes, 0); var tileMetaEndBytes = new byte[8]; stream.Read(tileMetaEndBytes, 0, tileMetaEndBytes.Length); var tileMetaEnd = BitConverter.ToInt64(tileMetaEndBytes, 0); // deserialize meta data. var metas = (SerializableGraphTileMetas)_runtimeTypeModel.Deserialize( new CappedStream(stream, stream.Position, tileMetaEnd - stream.Position), null, typeof(SerializableGraphTileMetas)); // create the datasource. var routerDataSource = new CHEdgeDataDataSource(stream, decompress, vehicles, metas, Zoom, this, 1000); if (!lazy) { // pre-load everything. for (int tileIdx = 0; tileIdx < metas.TileX.Length; tileIdx++) { routerDataSource.LoadMissingTile( new Tile(metas.TileX[tileIdx], metas.TileY[tileIdx], Zoom)); } } // return router datasource. return(routerDataSource); }
public void Seek_Origin_End_InLimit() { var source = CreateSourceStream(); const int stream_offset = 5; const int stream_length = 10; const long offset = -2; const long expected_position = stream_length + offset; const long expected_source_position = stream_length + offset + stream_offset; var limited = new LimitedStream(source, stream_offset, stream_length); var seek_actual = limited.Seek(offset, SeekOrigin.End); var actual_position = limited.Position; Assert.That.Value(seek_actual).IsEqual(expected_position); Assert.That.Value(actual_position).IsEqual(expected_position); Assert.That.Value(source.Position).IsEqual(expected_source_position); }
public void Seek_Origin_Begin_InLimit() { var source = CreateSourceStream(); const int stream_offset = 5; const int stream_length = 10; const long seek = 2; const long expected_seek = seek; var limited = new LimitedStream(source, stream_offset, stream_length); var seek_actual = limited.Seek(seek, SeekOrigin.Begin); var actual_position = limited.Position; var actual_source_position = source.Position; Assert.That.Value(seek_actual).IsEqual(expected_seek); Assert.That.Value(actual_position).IsEqual(expected_seek); Assert.That.Value(actual_source_position).IsEqual(stream_offset + expected_seek); }
/// <summary> /// Does the v1 serialization. /// </summary> /// <param name="stream"></param> /// <param name="graph"></param> /// <returns></returns> protected override void DoSerialize(LimitedStream stream, DynamicGraphRouterDataSource <LiveEdge> graph) { // create an index per tile. var dataPerTile = new Dictionary <Tile, UnserializedTileData>(); for (uint vertex = 1; vertex < graph.VertexCount + 1; vertex++) { // loop over all vertices and serialize all into the correct tile. float latitude, longitude; if (graph.GetVertex(vertex, out latitude, out longitude)) { // the vertex was found. // build the correct tile. Tile tile = Tile.CreateAroundLocation(new GeoCoordinate(latitude, longitude), Zoom); UnserializedTileData serializableGraphTile; if (!dataPerTile.TryGetValue(tile, out serializableGraphTile)) { // create the new tile. serializableGraphTile = new UnserializedTileData(); serializableGraphTile.Ids = new List <uint>(); serializableGraphTile.Latitude = new List <ushort>(); serializableGraphTile.Longitude = new List <ushort>(); serializableGraphTile.StringTable = new Dictionary <string, int>(); serializableGraphTile.Arcs = new List <SerializableGraphArcs>(); dataPerTile.Add(tile, serializableGraphTile); } // create short latitude/longitude. serializableGraphTile.Ids.Add(vertex); serializableGraphTile.Latitude.Add((ushort)(((tile.TopLeft.Latitude - latitude) / tile.Box.DeltaLat) * ushort.MaxValue)); serializableGraphTile.Longitude.Add((ushort)(((longitude - tile.TopLeft.Longitude) / tile.Box.DeltaLon) * ushort.MaxValue)); // get the arcs. KeyValuePair <uint, LiveEdge>[] arcs = graph.GetArcs(vertex); // serialize the arcs. if (arcs != null && arcs.Length > 0) { var serializableGraphArcs = new SerializableGraphArcs(); serializableGraphArcs.DestinationId = new uint[arcs.Length]; serializableGraphArcs.Forward = new bool[arcs.Length]; serializableGraphArcs.TileX = new int[arcs.Length]; serializableGraphArcs.TileY = new int[arcs.Length]; serializableGraphArcs.Tags = new SerializableTags[arcs.Length]; for (int idx = 0; idx < arcs.Length; idx++) { KeyValuePair <uint, LiveEdge> arc = arcs[idx]; // get destination tile. if (graph.GetVertex(arc.Key, out latitude, out longitude)) { // the destionation was found. Tile destinationTile = Tile.CreateAroundLocation( new GeoCoordinate(latitude, longitude), Zoom); serializableGraphArcs.DestinationId[idx] = arc.Key; serializableGraphArcs.TileX[idx] = destinationTile.X; serializableGraphArcs.TileY[idx] = destinationTile.Y; serializableGraphArcs.Forward[idx] = arc.Value.Forward; // get the tags. TagsCollectionBase tagsCollection = graph.TagsIndex.Get(arc.Value.Tags); if (tagsCollection != null) { serializableGraphArcs.Tags[idx] = new SerializableTags(); serializableGraphArcs.Tags[idx].Keys = new int[tagsCollection.Count]; serializableGraphArcs.Tags[idx].Values = new int[tagsCollection.Count]; int tagsIndex = 0; foreach (var tag in tagsCollection) { int key; if (!serializableGraphTile.StringTable.TryGetValue( tag.Key, out key)) { // string not yet in string table. key = serializableGraphTile.StringTable.Count; serializableGraphTile.StringTable.Add(tag.Key, key); } int value; if (!serializableGraphTile.StringTable.TryGetValue( tag.Value, out value)) { // string not yet in string table. value = serializableGraphTile.StringTable.Count; serializableGraphTile.StringTable.Add(tag.Value, value); } serializableGraphArcs.Tags[idx].Keys[tagsIndex] = key; serializableGraphArcs.Tags[idx].Values[tagsIndex] = value; tagsIndex++; } } } } serializableGraphTile.Arcs.Add(serializableGraphArcs); } } } // LAYOUT OF V2: {HEADER}{compressionflag(1byte)}{#tiles(4byte)}{tilesMetaEnd(8byte)}{tiles-meta-data-xxxxxxx}{tiles-data} // {HEADER} : already written before this method. // {#tiles(4byte)} : the number of tiles in this file (calculate the offset of the {tiles-data} // section using (TileMetaSize * dataPerTile.Count + 4 + 8) // {tilesMetaEnd(8byte)} : the end of the meta tiles. // {tiles-meta-data-xxxxxxx} : the serialized tile metadata. // {tiles-data} : the actual tile data. // calculate the space needed for the tile offset. const long tileMetaOffset = 1 + 4 + 8; long tileOffset = TileMetaSize * dataPerTile.Count + tileMetaOffset; // all tile metadata + a tile count + tags offset. // build the tile metadata while writing the tile data. stream.Seek(tileOffset, SeekOrigin.Begin); var metas = new SerializableGraphTileMetas(); metas.Length = new int[dataPerTile.Count]; metas.Offset = new long[dataPerTile.Count]; metas.TileX = new int[dataPerTile.Count]; metas.TileY = new int[dataPerTile.Count]; int metasIndex = 0; foreach (var unserializedTileData in dataPerTile) { // create the tile meta. metas.TileX[metasIndex] = unserializedTileData.Key.X; metas.TileY[metasIndex] = unserializedTileData.Key.Y; metas.Offset[metasIndex] = stream.Position; // create the tile. var serializableGraphTile = new SerializableGraphTile(); serializableGraphTile.Arcs = unserializedTileData.Value.Arcs.ToArray(); serializableGraphTile.Ids = unserializedTileData.Value.Ids.ToArray(); serializableGraphTile.Latitude = unserializedTileData.Value.Latitude.ToArray(); serializableGraphTile.Longitude = unserializedTileData.Value.Longitude.ToArray(); serializableGraphTile.StringTable = new string[unserializedTileData.Value.StringTable.Count]; foreach (var stringEntry in unserializedTileData.Value.StringTable) { serializableGraphTile.StringTable[stringEntry.Value] = stringEntry.Key; } // serialize the tile. if (!_compress) { // compresses the file. _runtimeTypeModel.Serialize(stream, serializableGraphTile); } else { // first compress the data, then write. var uncompressed = new MemoryStream(); _runtimeTypeModel.Serialize(uncompressed, serializableGraphTile); var uncompressedBuffer = uncompressed.ToArray(); byte[] compressed = GZipStream.CompressBuffer(uncompressedBuffer); stream.Write(compressed, 0, compressed.Length); } // calculate the length of the data that was just serialized. metas.Length[metasIndex] = (int)(stream.Position - metas.Offset[metasIndex]); metasIndex++; } // serialize all tile meta data. stream.Seek(tileMetaOffset, SeekOrigin.Begin); _runtimeTypeModel.Serialize(stream, metas); long tileMetaEnd = stream.Position; // save the meta and. // save all the offsets. stream.Seek(0, SeekOrigin.Begin); byte[] compressionFlag = new[] { (byte)(_compress ? 1 : 0) }; stream.Write(compressionFlag, 0, 1); byte[] tileCountBytes = BitConverter.GetBytes(metas.TileX.Length); stream.Write(tileCountBytes, 0, tileCountBytes.Length); // 4 bytes byte[] tileMetaEndBytes = BitConverter.GetBytes(tileMetaEnd); stream.Write(tileMetaEndBytes, 0, tileMetaEndBytes.Length); // 8 bytes stream.Flush(); }
/// <summary> /// Does the v2 serialization. /// </summary> /// <param name="stream"></param> /// <param name="graph"></param> /// <returns></returns> protected override void DoSerialize(LimitedStream stream, DynamicGraphRouterDataSource <CHEdgeData> graph) { // sort the graph. IDynamicGraph <CHEdgeData> sortedGraph = CHEdgeDataDataSourceSerializer.SortGraph(graph); // create the regions. SortedDictionary <ulong, List <uint> > regions = new SortedDictionary <ulong, List <uint> >(); for (uint newVertexId = 1; newVertexId < sortedGraph.VertexCount + 1; newVertexId++) { // add to the CHRegions. float latitude, longitude; sortedGraph.GetVertex(newVertexId, out latitude, out longitude); Tile tile = Tile.CreateAroundLocation(new GeoCoordinate( latitude, longitude), RegionZoom); List <uint> regionVertices; if (!regions.TryGetValue(tile.Id, out regionVertices)) { regionVertices = new List <uint>(); regions.Add(tile.Id, regionVertices); } regionVertices.Add(newVertexId); } // serialize the sorted graph. // [START_OF_BLOCKS][[SIZE_OF_REGION_INDEX][REGION_INDEX][REGIONS]][[SIZE_OF_BLOCK_INDEX][BLOCK_INDEX][BLOCKS]] // STRART_OF_BLOCKS: 4bytes // SIZE_OF_REGION_INDEX: 4bytes // REGION_INDEX: see SIZE_OF_REGION_INDEX // REGIONS: see START_OF_BLOCKS - 4bytes // SIZE_OF_BLOCK_INDEX: 4bytes // BLOCK_INDEX: see SIZE_OF_BLOCK_INDEX. // BLOCKS: from START_OF_BLOCKS + 4bytes + SIZE_OF_BLOCKS_INDEX until END. // serialize regions and build their index. CHVertexRegionIndex chRegionIndex = new CHVertexRegionIndex(); chRegionIndex.LocationIndex = new int[regions.Count]; chRegionIndex.RegionIds = new ulong[regions.Count]; var memoryStream = new MemoryStream(); int regionIdx = 0; foreach (KeyValuePair <ulong, List <uint> > region in regions) { // serialize. CHVertexRegion chRegion = new CHVertexRegion(); chRegion.Vertices = region.Value.ToArray(); _runtimeTypeModel.Serialize(memoryStream, chRegion); // set index. chRegionIndex.LocationIndex[regionIdx] = (int)memoryStream.Position; chRegionIndex.RegionIds[regionIdx] = region.Key; regionIdx++; } stream.Seek(8, SeekOrigin.Begin); // move to beginning of [REGION_INDEX] _runtimeTypeModel.Serialize(stream, chRegionIndex); // write region index. int sizeRegionIndex = (int)(stream.Position - 8); // now at beginning of [REGIONS] memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.WriteTo(stream); // write regions. memoryStream.Dispose(); int startOfBlocks = (int)stream.Position; // now at beginning of [SIZE_OF_BLOCK_INDEX] stream.Seek(0, SeekOrigin.Begin); // move to beginning stream.Write(BitConverter.GetBytes(startOfBlocks), 0, 4); // write start position of blocks. Now at [SIZE_OF_REGION_INDEX] stream.Write(BitConverter.GetBytes(sizeRegionIndex), 0, 4); // write size of blocks index. // serialize the blocks and build their index. memoryStream = new MemoryStream(); List <int> blockLocations = new List <int>(); uint vertexId = 1; while (vertexId < sortedGraph.VertexCount) { uint blockId = vertexId; List <CHArc> blockArcs = new List <CHArc>(); List <CHVertex> blockVertices = new List <CHVertex>(); while (vertexId < blockId + BlockVertexSize && vertexId < sortedGraph.VertexCount + 1) { // create this block. CHVertex chVertex = new CHVertex(); float latitude, longitude; sortedGraph.GetVertex(vertexId, out latitude, out longitude); chVertex.Latitude = latitude; chVertex.Longitude = longitude; chVertex.ArcIndex = (ushort)(blockArcs.Count); foreach (KeyValuePair <uint, CHEdgeData> sortedArc in sortedGraph.GetArcs(vertexId)) { CHArc chArc = new CHArc(); chArc.TargetId = sortedArc.Key; chArc.ShortcutId = sortedArc.Value.ContractedVertexId; chArc.Weight = sortedArc.Value.Weight; chArc.Direction = sortedArc.Value.Direction; blockArcs.Add(chArc); } chVertex.ArcCount = (ushort)(blockArcs.Count - chVertex.ArcIndex); blockVertices.Add(chVertex); vertexId++; // move to the next vertex. } // create block. CHBlock block = new CHBlock(); block.Arcs = blockArcs.ToArray(); block.Vertices = blockVertices.ToArray(); // TODO: get rid of the list and create an array to begin with. // write blocks. _runtimeTypeModel.Serialize(memoryStream, block); blockLocations.Add((int)memoryStream.Position); } CHBlockIndex blockIndex = new CHBlockIndex(); blockIndex.LocationIndex = blockLocations.ToArray(); stream.Seek(startOfBlocks + 4, SeekOrigin.Begin); // move to beginning of [BLOCK_INDEX] _runtimeTypeModel.Serialize(stream, blockIndex); // write region index. int sizeBlockIndex = (int)(stream.Position - (startOfBlocks + 4)); // now at beginning of [BLOCKS] memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.WriteTo(stream); // write blocks. memoryStream.Dispose(); stream.Seek(startOfBlocks, SeekOrigin.Begin); // move to [SIZE_OF_BLOCK_INDEX] stream.Write(BitConverter.GetBytes(sizeBlockIndex), 0, 4); // write start position of blocks. Now at [SIZE_OF_REGION_INDEX] stream.Flush(); }
/// <summary> /// Does the v2 serialization. /// </summary> /// <param name="stream"></param> /// <param name="graph"></param> /// <returns></returns> protected override void DoSerialize(LimitedStream stream, DynamicGraphRouterDataSource <CHEdgeData> graph) { // sort the graph. var sortedGraph = graph; // create the regions. var regions = new SortedDictionary <ulong, List <uint> >(); for (uint newVertexId = 1; newVertexId < sortedGraph.VertexCount + 1; newVertexId++) { // add to the CHRegions. float latitude, longitude; sortedGraph.GetVertex(newVertexId, out latitude, out longitude); var tile = Tile.CreateAroundLocation(new GeoCoordinate( latitude, longitude), _regionZoom); List <uint> regionVertices; if (!regions.TryGetValue(tile.Id, out regionVertices)) { regionVertices = new List <uint>(); regions.Add(tile.Id, regionVertices); } regionVertices.Add(newVertexId); } // serialize the sorted graph. // [START_OF_BLOCKS][START_OF_SHAPES][START_OF_REVERSE][START_OF_TAGS][[SIZE_OF_REGION_INDEX][REGION_INDEX][REGIONS]] // [[SIZE_OF_BLOCK_INDEX][BLOCK_INDEX][BLOCKS]] // [[SIZE_OF_SHAPE_INDEX][SHAPE_INDEX][SHAPES]] // [[SIZE_OF_REVERSE_INDEX][REVERSE_INDEX][REVERSE]] // [TAGS] // START_OF_BLOCKS: 4bytes // START_OF_SHAPES: 4bytes // START_OF_STAGS: 4bytes // START_OF_REVERSE 4bytes // SIZE_OF_REGION_INDEX: 4bytes // REGION_INDEX: see SIZE_OF_REGION_INDEX // REGIONS: see START_OF_BLOCKS - 4bytes // SIZE_OF_BLOCK_INDEX: 4bytes // BLOCK_INDEX: see SIZE_OF_BLOCK_INDEX. // BLOCKS: from START_OF_BLOCKS + 4bytes + SIZE_OF_BLOCKS_INDEX until END. // serialize regions and build their index. var chRegionIndex = new CHVertexRegionIndex(); chRegionIndex.LocationIndex = new int[regions.Count]; chRegionIndex.RegionIds = new ulong[regions.Count]; var memoryStream = new MemoryStream(); int regionIdx = 0; foreach (var region in regions) { // serialize. var chRegion = new CHVertexRegion(); chRegion.Vertices = region.Value.ToArray(); _runtimeTypeModel.Serialize(memoryStream, chRegion); // set index. chRegionIndex.LocationIndex[regionIdx] = (int)memoryStream.Position; chRegionIndex.RegionIds[regionIdx] = region.Key; regionIdx++; } stream.Seek(INDEX_SIZE, SeekOrigin.Begin); // move to beginning of [REGION_INDEX] _runtimeTypeModel.Serialize(stream, chRegionIndex); // write region index. var sizeRegionIndex = (int)(stream.Position - INDEX_SIZE); // now at beginning of [REGIONS] memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.WriteTo(stream); // write regions. memoryStream.Dispose(); var startOfBlocks = (int)stream.Position; // now at beginning of [SIZE_OF_BLOCK_INDEX] // serialize the blocks and build their index. memoryStream = new MemoryStream(); var blocks = new List <CHBlock>(); var blockLocations = new List <int>(); var blockShapeLocations = new List <int>(); var blockReverseLocations = new List <int>(); var blockShapes = new List <CHBlockCoordinates>(); var reverseNeighbours = new Dictionary <uint, List <uint> >(); uint vertexId = 1; while (vertexId < sortedGraph.VertexCount + 1) { uint blockId = vertexId; var blockArcs = new List <CHArc>(); var blockArcCoordinates = new List <CHArcCoordinates>(); var blockVertices = new List <CHVertex>(); while (vertexId < blockId + _blockVertexSize && vertexId < sortedGraph.VertexCount + 1) { // create this block. var chVertex = new CHVertex(); float latitude, longitude; sortedGraph.GetVertex(vertexId, out latitude, out longitude); chVertex.Latitude = latitude; chVertex.Longitude = longitude; chVertex.ArcIndex = (ushort)(blockArcs.Count); List <uint> reverse = null; foreach (var sortedArc in sortedGraph.GetEdges(vertexId)) { var chArc = new CHArc(); chArc.TargetId = sortedArc.Neighbour; chArc.Meta = sortedArc.EdgeData.Meta; chArc.Value = sortedArc.EdgeData.Value; chArc.Weight = sortedArc.EdgeData.Weight; blockArcs.Add(chArc); var chArcCoordinates = new CHArcCoordinates(); chArcCoordinates.Coordinates = sortedArc.Intermediates.ToSimpleArray(); blockArcCoordinates.Add(chArcCoordinates); if (sortedArc.EdgeData.RepresentsNeighbourRelations) { if (!reverseNeighbours.TryGetValue(sortedArc.Neighbour, out reverse)) { reverse = new List <uint>(); reverseNeighbours[sortedArc.Neighbour] = reverse; } reverse.Add(vertexId); } } chVertex.ArcCount = (ushort)(blockArcs.Count - chVertex.ArcIndex); blockVertices.Add(chVertex); vertexId++; // move to the next vertex. } // create block. var block = new CHBlock(); block.Arcs = blockArcs.ToArray(); block.Vertices = blockVertices.ToArray(); // TODO: get rid of the list and create an array to begin with. blocks.Add(block); var blockCoordinates = new CHBlockCoordinates(); blockCoordinates.Arcs = blockArcCoordinates.ToArray(); // write blocks. var blockStream = new MemoryStream(); _runtimeTypeModel.Serialize(blockStream, block); var compressed = GZipStream.CompressBuffer(blockStream.ToArray()); blockStream.Dispose(); memoryStream.Write(compressed, 0, compressed.Length); blockLocations.Add((int)memoryStream.Position); blockShapes.Add(blockCoordinates); } var blockIndex = new CHBlockIndex(); blockIndex.BlockLocationIndex = blockLocations.ToArray(); stream.Seek(startOfBlocks + 4, SeekOrigin.Begin); // move to beginning of [BLOCK_INDEX] _runtimeTypeModel.Serialize(stream, blockIndex); // write block index. int sizeBlockIndex = (int)(stream.Position - (startOfBlocks + 4)); // now at beginning of [BLOCKS] memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.WriteTo(stream); // write blocks. memoryStream.Dispose(); // write shapes and index. memoryStream = new MemoryStream(); var startOfShapes = stream.Position; foreach (var blockCoordinates in blockShapes) { // write shape blocks. var blockShapeStream = new MemoryStream(); _runtimeTypeModel.Serialize(blockShapeStream, blockCoordinates); var compressed = GZipStream.CompressBuffer(blockShapeStream.ToArray()); blockShapeStream.Dispose(); memoryStream.Write(compressed, 0, compressed.Length); blockShapeLocations.Add((int)memoryStream.Position); } blockIndex = new CHBlockIndex(); blockIndex.BlockLocationIndex = blockShapeLocations.ToArray(); stream.Seek(startOfShapes + 4, SeekOrigin.Begin); // move to beginning of [SHAPE_INDEX] _runtimeTypeModel.Serialize(stream, blockIndex); // write shape index. int sizeShapeIndex = (int)(stream.Position - (startOfShapes + 4)); // now at beginning of [SHAPE] memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.WriteTo(stream); // write shapes. memoryStream.Dispose(); // build reverse blocks. var blockReverses = new List <CHBlockReverse>(); List <uint> storedReverses = null; uint currentVertexId = 0; for (int i = 0; i < blocks.Count; i++) { var blockReverse = new CHBlockReverse(); blockReverse.Vertices = new CHArcReverse[blocks[i].Vertices.Length]; for (int j = 0; j < blocks[i].Vertices.Length; j++) { currentVertexId++; var blockReverseArc = new CHArcReverse(); blockReverseArc.Neighbours = new uint[0]; if (reverseNeighbours.TryGetValue(currentVertexId, out storedReverses)) { blockReverseArc.Neighbours = storedReverses.ToArray(); } blockReverse.Vertices[j] = blockReverseArc; } blockReverses.Add(blockReverse); } // write reverse and index. memoryStream = new MemoryStream(); var startOfReverses = stream.Position; foreach (var blockReverse in blockReverses) { // write reverse blocks. var blockReverseStream = new MemoryStream(); _runtimeTypeModel.Serialize(blockReverseStream, blockReverse); var compressed = GZipStream.CompressBuffer(blockReverseStream.ToArray()); blockReverseStream.Dispose(); memoryStream.Write(compressed, 0, compressed.Length); blockReverseLocations.Add((int)memoryStream.Position); } blockIndex = new CHBlockIndex(); blockIndex.BlockLocationIndex = blockReverseLocations.ToArray(); stream.Seek(startOfReverses + 4, SeekOrigin.Begin); // move to beginning of [REVERSE_INDEX] _runtimeTypeModel.Serialize(stream, blockIndex); // write reverse index. int sizeReverseIndex = (int)(stream.Position - (startOfReverses + 4)); // now at beginning of [REVERSE] memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.WriteTo(stream); // write reverses. memoryStream.Dispose(); // write tags after reverses. int tagsPosition = (int)stream.Position; TagIndexSerializer.SerializeBlocks(stream, graph.TagsIndex, 10); stream.Seek(startOfBlocks, SeekOrigin.Begin); // move to [SIZE_OF_BLOCK_INDEX] stream.Write(BitConverter.GetBytes(sizeBlockIndex), 0, 4); // write start position of blocks. stream.Seek(startOfShapes, SeekOrigin.Begin); // move to [SIZE_OF_BLOCK_INDEX] stream.Write(BitConverter.GetBytes(sizeShapeIndex), 0, 4); // write start position of blocks. stream.Seek(startOfReverses, SeekOrigin.Begin); // move to [SIZE_OF_REVERSE_INDEX] stream.Write(BitConverter.GetBytes(sizeReverseIndex), 0, 4); // write start position of blocks. // write index. stream.Seek(0, SeekOrigin.Begin); // move to beginning stream.Write(BitConverter.GetBytes(startOfBlocks), 0, 4); // write start position of blocks. stream.Write(BitConverter.GetBytes(startOfShapes), 0, 4); // write start position of shapes. stream.Write(BitConverter.GetBytes(startOfReverses), 0, 4); // write start position of shapes. stream.Write(BitConverter.GetBytes(tagsPosition), 0, 4); stream.Write(BitConverter.GetBytes(sizeRegionIndex), 0, 4); // write size of region index. stream.Flush(); }