/// <summary> /// Does the v2 deserialization. /// </summary> /// <param name="stream"></param> /// <param name="lazy"></param> /// <param name="vehicles"></param> /// <returns></returns> protected override IBasicRouterDataSource <CHEdgeData> DoDeserialize( LimitedStream stream, bool lazy, IEnumerable <string> vehicles) { var intBytes = new byte[4]; stream.Read(intBytes, 0, 4); int startOfBlocks = BitConverter.ToInt32(intBytes, 0); stream.Read(intBytes, 0, 4); int startOfTags = BitConverter.ToInt32(intBytes, 0); stream.Read(intBytes, 0, 4); int sizeRegionIndex = BitConverter.ToInt32(intBytes, 0); // deserialize regions index. var chVertexRegionIndex = (CHVertexRegionIndex)_runtimeTypeModel.Deserialize( new CappedStream(stream, stream.Position, sizeRegionIndex), null, typeof(CHVertexRegionIndex)); // deserialize blocks index. stream.Seek(startOfBlocks, SeekOrigin.Begin); stream.Read(intBytes, 0, 4); int sizeBlockIndex = BitConverter.ToInt32(intBytes, 0); var chBlockIndex = (CHBlockIndex)_runtimeTypeModel.Deserialize( new CappedStream(stream, stream.Position, sizeBlockIndex), null, typeof(CHBlockIndex)); // deserialize tags. stream.Seek(startOfTags, SeekOrigin.Begin); ITagsCollectionIndexReadonly tagsIndex = TagIndexSerializer.DeserializeBlocks(stream); return(new v2.CHEdgeDataDataSource(stream, this, vehicles, sizeRegionIndex + 12, chVertexRegionIndex, _regionZoom, startOfBlocks + sizeBlockIndex + 4, chBlockIndex, (uint)_blockVertexSize, tagsIndex)); }
/// <summary> /// Creates a new osm memory router data source. /// </summary> /// <param name="graph"></param> /// <param name="tagsIndex"></param> /// <exception cref="ArgumentNullException"></exception> public DynamicGraphRouterDataSource(IDynamicGraph <TEdgeData> graph, ITagsCollectionIndexReadonly tagsIndex) { if (graph == null) { throw new ArgumentNullException("graph"); } if (tagsIndex == null) { throw new ArgumentNullException("tagsIndex"); } _graph = graph; _vertexIndex = new QuadTree <GeoCoordinate, uint>(); _tagsIndex = tagsIndex; _supportedVehicles = new HashSet <Vehicle>(); // add the current graph's vertices to the vertex index. for (uint newVertexId = 1; newVertexId < graph.VertexCount + 1; newVertexId++) { // add to the CHRegions. float latitude, longitude; graph.GetVertex(newVertexId, out latitude, out longitude); _vertexIndex.Add(new GeoCoordinate(latitude, longitude), newVertexId); } }
public void TestRandomBlockTagSerializatonNonBeginPositionLimitedStreamRegression() { SimpleTagsCollectionIndex tagsIndex = new SimpleTagsCollectionIndex(); SimpleTagsCollection tagsCollection = new SimpleTagsCollection(); for (int i = 0; i < 101; i++) { int tagCollectionSize = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10) + 1; for (int idx = 0; idx < tagCollectionSize; idx++) { int tagValue = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10); tagsCollection.Add( string.Format("key_{0}", tagValue), string.Format("value_{0}", tagValue)); } uint tagsId = tagsIndex.Add(tagsCollection); } ITagsCollectionIndexReadonly tagsIndexReadonly = this.SerializeDeserializeBlockLimitedStream(tagsIndex, 10, 123); Assert.AreEqual(tagsIndex.Max, tagsIndexReadonly.Max); for (uint idx = 0; idx < tagsIndex.Max; idx++) { this.CompareTagsCollections(tagsIndex.Get(idx), tagsIndexReadonly.Get(idx)); } }
/// <summary> /// Executes the tests. /// </summary> public static void Test() { TagsTableCollectionIndex index = new TagsTableCollectionIndex(); // first fill the index. ITagCollectionIndexTests.FillIndex(index, 100000); // serialize the index. FileInfo testFile = new FileInfo(@"test.file"); testFile.Delete(); Stream writeStream = testFile.OpenWrite(); OsmSharp.Logging.Log.TraceEvent("Blocked", OsmSharp.Logging.TraceEventType.Information, "Serializing blocked file...."); TagIndexSerializer.SerializeBlocks(writeStream, index, 100); writeStream.Flush(); writeStream.Dispose(); OsmSharp.Logging.Log.TraceEvent("Blocked", OsmSharp.Logging.TraceEventType.Information, string.Format("Serialized file: {0}KB", testFile.Length / 1024)); // deserialize the index. Stream readStream = testFile.OpenRead(); ITagsCollectionIndexReadonly readOnlyIndex = TagIndexSerializer.DeserializeBlocks(readStream); // test access. OsmSharp.Logging.Log.TraceEvent("Blocked", OsmSharp.Logging.TraceEventType.Information, "Started testing random access...."); ITagCollectionIndexTests.TestRandomAccess("Blocked", readOnlyIndex, 1000); readStream.Dispose(); testFile.Delete(); }
public void TestRandomBlockTagSerializaton() { TagsTableCollectionIndex tagsIndex = new TagsTableCollectionIndex(); TagsCollection tagsCollection = new TagsCollection(); for (int i = 0; i < 100; i++) { int tagCollectionSize = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10) + 1; for (int idx = 0; idx < tagCollectionSize; idx++) { int tagValue = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10); tagsCollection.Add( string.Format("key_{0}", tagValue), string.Format("value_{0}", tagValue)); } uint tagsId = tagsIndex.Add(tagsCollection); } ITagsCollectionIndexReadonly tagsIndexReadonly = this.SerializeDeserializeBlock(tagsIndex, 10, 0); Assert.AreEqual(tagsIndex.Max, tagsIndexReadonly.Max); for (uint idx = 0; idx < tagsIndex.Max; idx++) { ComparisonHelpers.CompareTags(tagsIndex.Get(idx), tagsIndexReadonly.Get(idx)); } }
/// <summary> /// Creates a new osm memory router data source. /// </summary> /// <exception cref="ArgumentNullException"></exception> public DynamicGraphRouterDataSource(ITagsCollectionIndexReadonly tagsIndex, int initSize) { if (tagsIndex == null) { throw new ArgumentNullException("tagsIndex"); } _graph = new MemoryDynamicGraph <TEdgeData>(initSize); _vertexIndex = null; // do not create an index initially. _tagsIndex = tagsIndex; _supportedVehicles = new HashSet <Vehicle>(); }
/// <summary> /// Creates a new osm memory router data source. /// </summary> /// <exception cref="ArgumentNullException"></exception> public DynamicGraphRouterDataSource(ITagsCollectionIndexReadonly tagsIndex) { if (tagsIndex == null) { throw new ArgumentNullException("tagsIndex"); } _graph = new MemoryDynamicGraph <TEdgeData>(); _vertexIndex = new QuadTree <GeoCoordinate, uint>(); _tagsIndex = tagsIndex; _supportedVehicles = new HashSet <Vehicle>(); }
/// <summary> /// Creates a new CH edge data source. /// </summary> public CHEdgeDataDataSource(Stream stream, CHEdgeDataDataSourceSerializer serializer, int startOfRegions, CHVertexRegionIndex regionIndex, int zoom, int startOfBlocks, CHBlockIndex blockIndex, uint blockSize, ITagsCollectionIndexReadonly tagsIndex) { _stream = stream; _serializer = serializer; this.InitializeRegions(startOfRegions, regionIndex, zoom); this.InitializeBlocks(startOfBlocks, blockIndex, blockSize); _blocks = new LRUCache <uint, CHBlock>(1000); _regions = new LRUCache <ulong, CHVertexRegion>(1000); _tagsIndex = tagsIndex; }
/// <summary> /// Tests random access to an already filled index. /// </summary> /// <param name="name"></param> /// <param name="index"></param> /// <param name="testCount"></param> public static void TestRandomAccess(string name, ITagsCollectionIndexReadonly index, int testCount) { PerformanceInfoConsumer performanceInfo = new PerformanceInfoConsumer(string.Format("{0}.Access", name)); performanceInfo.Start(); performanceInfo.Report("Random accessing tags index {0} times...", testCount); for (int idx = 0; idx < testCount; idx++) { uint collectionIndex = (uint)OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(index.Max); TagsCollectionBase collection = index.Get(collectionIndex); } performanceInfo.Stop(); }
public void TestSimpleTagSerializatonLimitedStreamRegression() { TagsTableCollectionIndex tagsIndex = new TagsTableCollectionIndex(); TagsCollection tagsCollection = new TagsCollection(); tagsCollection.Add("key1", "value1"); uint tagsId = tagsIndex.Add(tagsCollection); ITagsCollectionIndexReadonly tagsIndexReadonly = this.SerializeDeserializeLimitedStream(tagsIndex, 132); Assert.AreEqual(tagsIndex.Max, tagsIndexReadonly.Max); for (uint idx = 0; idx < tagsIndex.Max; idx++) { ComparisonHelpers.CompareTags(tagsIndex.Get(idx), tagsIndexReadonly.Get(idx)); } }
public void TestSimpleTagSerializatonNonBeginPosition() { SimpleTagsCollectionIndex tagsIndex = new SimpleTagsCollectionIndex(); SimpleTagsCollection tagsCollection = new SimpleTagsCollection(); tagsCollection.Add("key1", "value1"); uint tagsId = tagsIndex.Add(tagsCollection); ITagsCollectionIndexReadonly tagsIndexReadonly = this.SerializeDeserialize(tagsIndex, 1201); Assert.AreEqual(tagsIndex.Max, tagsIndexReadonly.Max); for (uint idx = 0; idx < tagsIndex.Max; idx++) { this.CompareTagsCollections(tagsIndex.Get(idx), tagsIndexReadonly.Get(idx)); } }
/// <summary> /// Serializes the meta-data. /// </summary> /// <param name="stream"></param> /// <param name="tagsCollectionIndex"></param> protected virtual void SerializeTags(LimitedStream stream, ITagsCollectionIndexReadonly tagsCollectionIndex) { RuntimeTypeModel typeModel = RuntimeTypeModel.Create(); typeModel.Add(typeof(SerializableTag), true); // write tags collection-count. var countBytes = BitConverter.GetBytes(tagsCollectionIndex.Max); stream.Write(countBytes, 0, 4); int blockSize = 1000; var tagsQueue = new List <SerializableTag>(); // serialize tags collections one-by-one. for (uint idx = 0; idx < tagsCollectionIndex.Max; idx++) { // serialize objects one-by-one. var tagsCollection = tagsCollectionIndex.Get(idx); foreach (var tag in tagsCollection) { tagsQueue.Add(new SerializableTag() { CollectionId = idx, Key = tag.Key, Value = tag.Value }); if (tagsQueue.Count == blockSize) { // time to serialize. typeModel.SerializeWithSize(stream, tagsQueue.ToArray()); tagsQueue.Clear(); } } } tagsQueue.Add(new SerializableTag() { CollectionId = int.MaxValue, Key = string.Empty, Value = string.Empty }); typeModel.SerializeWithSize(stream, tagsQueue.ToArray()); }
/// <summary> /// Deserializes a block. /// </summary> /// <param name="blockIdx"></param> private void DeserializeBlock(int blockIdx) { // calculate current bounds. _currentBlockMin = (uint)(blockIdx * _blockSize); // move stream to correct position. int blockOffset = 0; if (blockIdx > 0) { blockOffset = _blockPositions[blockIdx - 1]; } // seek stream. _stream.Seek(blockOffset + _begin, SeekOrigin.Begin); // deserialize this block. Ionic.Zlib.GZipStream gzipStream = new Ionic.Zlib.GZipStream(_stream, Ionic.Zlib.CompressionMode.Decompress); _currentBlock = TagIndexSerializer.Deserialize(gzipStream); }
/// <summary> /// Serializes the tags into different indexed blocks of given size. /// </summary> /// <param name="stream"></param> /// <param name="tagsIndex"></param> /// <param name="blockSize"></param> public static void SerializeBlocks(Stream stream, ITagsCollectionIndexReadonly tagsIndex, uint blockSize) { int begin = (int)stream.Position; // calculate the amount of blocks. uint blocks = (uint)System.Math.Ceiling((float)tagsIndex.Max / (float)blockSize); // store block count. stream.Write(BitConverter.GetBytes((int)blocks), 0, 4); // store block size. stream.Write(BitConverter.GetBytes((int)blockSize), 0, 4); // move the stream to make room for the index. stream.Seek((blocks + 2) * 4 + begin, SeekOrigin.Begin); int beginBlocks = (int)stream.Position; // keep looping over these blocks. int[] blockPositions = new int[blocks]; for (uint blockIdx = 0; blockIdx < blocks; blockIdx++) { uint from = blockIdx * blockSize; uint to = from + blockSize; MemoryStream memoryStream = new MemoryStream(); TagIndexSerializer.Serialize(memoryStream, tagsIndex, from, to); byte[] compressed = Ionic.Zlib.GZipStream.CompressBuffer(memoryStream.ToArray()); memoryStream.Dispose(); stream.Write(compressed, 0, compressed.Length); blockPositions[blockIdx] = (int)stream.Position - beginBlocks; } // write the block positions. stream.Seek(begin + 8, SeekOrigin.Begin); for (int blockIdx = 0; blockIdx < blocks; blockIdx++) { stream.Write(BitConverter.GetBytes(blockPositions[blockIdx]), 0, 4); } }
/// <summary> /// Returns the tags collection. /// </summary> /// <param name="tagsId"></param> /// <returns></returns> public TagsCollectionBase Get(uint tagsId) { // check bounds of current block. if (_currentBlock != null) { if (tagsId >= _currentBlockMin && tagsId < (_currentBlockMin + _blockSize)) { // tag is in current block. return(_currentBlock.Get(tagsId)); } } // load another block. int blockIdx = (int)System.Math.Floor((float)tagsId / (float)_blockSize); // check if outside of the scope of this index. if (blockIdx >= _blockPositions.Length) { return(new TagsCollection()); } // check cache. KeyValuePair <uint, ITagsCollectionIndexReadonly> blockOut; if (!_cache.TryGet(blockIdx, out blockOut)) { // cache miss! // deserialize block. this.DeserializeBlock(blockIdx); _cache.Add(blockIdx, new KeyValuePair <uint, ITagsCollectionIndexReadonly>( _currentBlockMin, _currentBlock)); } else { // cache hit! _currentBlock = blockOut.Value; _currentBlockMin = blockOut.Key; } return(_currentBlock.Get(tagsId)); }
/// <summary> /// Serializes the tags between the given two indexes. /// </summary> /// <param name="stream"></param> /// <param name="tagsIndex"></param> /// <param name="from"></param> /// <param name="to"></param> public static void Serialize(Stream stream, ITagsCollectionIndexReadonly tagsIndex, uint from, uint to) { int begin = (int)stream.Position; // limit to tagsIndex count. if (tagsIndex.Max < to) { to = tagsIndex.Max; } // build a string index. ObjectTable <string> stringTable = new ObjectTable <string>(false); // convert tag collections to simpler objects. List <KeyValuePair <uint, List <KeyValuePair <uint, uint> > > > tagsIndexList = new List <KeyValuePair <uint, List <KeyValuePair <uint, uint> > > >(); for (uint tagId = from; tagId < to; tagId++) { TagsCollectionBase tagsCollection = tagsIndex.Get(tagId); if (tagsCollection != null) { // convert the tags collection to a list and add to the tag index. List <KeyValuePair <uint, uint> > tagsList = new List <KeyValuePair <uint, uint> >(); foreach (Tag tag in tagsCollection) { uint keyId = stringTable.Add(tag.Key); uint valueId = stringTable.Add(tag.Value); tagsList.Add(new KeyValuePair <uint, uint>( keyId, valueId)); } tagsIndexList.Add(new KeyValuePair <uint, List <KeyValuePair <uint, uint> > >(tagId, tagsList)); } } // do the serialization. TagIndexSerializer.Serialize(begin, stream, tagsIndexList, stringTable); // clear everything. tagsIndexList.Clear(); }
/// <summary> /// Creates a new CH edge data source. /// </summary> public CHEdgeDataDataSource(Stream stream, CHEdgeDataDataSourceSerializer serializer, IEnumerable <string> vehicles, int startOfRegions, CHVertexRegionIndex regionIndex, int zoom, int startOfBlocks, CHBlockIndex blockIndex, uint blockSize, int startOfShapes, CHBlockIndex shapeIndex, int startOfReverses, CHBlockIndex reversesIndex, ITagsCollectionIndexReadonly tagsIndex) { _stream = stream; _serializer = serializer; _vehicles = new HashSet <string>(vehicles); this.InitializeRegions(startOfRegions, regionIndex, zoom); this.InitializeBlocks(startOfBlocks, blockIndex, blockSize); this.InitializeShapes(startOfShapes, shapeIndex); this.InitializeReverses(startOfReverses, reversesIndex); _blocks = new LRUCache <uint, CHBlock>(5000); _blockShapes = new LRUCache <uint, CHBlockCoordinates>(1000); _blockReverses = new LRUCache <uint, CHBlockReverse>(1000); _regions = new LRUCache <ulong, CHVertexRegion>(1000); _tagsIndex = tagsIndex; }
public void TestRandomPartialTagSerializaton() { SimpleTagsCollectionIndex tagsIndex = new SimpleTagsCollectionIndex(); SimpleTagsCollection tagsCollection = new SimpleTagsCollection(); for (int i = 0; i < 100; i++) { int tagCollectionSize = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10) + 1; for (int idx = 0; idx < tagCollectionSize; idx++) { int tagValue = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10); tagsCollection.Add( string.Format("key_{0}", tagValue), string.Format("value_{0}", tagValue)); } uint tagsId = tagsIndex.Add(tagsCollection); } uint from = 40; uint to = 50; ITagsCollectionIndexReadonly tagsIndexReadonly = this.SerializeDeserialize(tagsIndex, from, to); Assert.AreEqual(System.Math.Min(to, tagsIndex.Max), tagsIndexReadonly.Max); for (uint idx = 0; idx < tagsIndex.Max; idx++) { if (idx >= from && idx < to) { this.CompareTagsCollections(tagsIndex.Get(idx), tagsIndexReadonly.Get(idx)); } else { Assert.IsNull(tagsIndexReadonly.Get(idx)); } } from = 0; to = 100; tagsIndexReadonly = this.SerializeDeserialize(tagsIndex, from, to); Assert.AreEqual(System.Math.Min(to, tagsIndex.Max), tagsIndexReadonly.Max); for (uint idx = 0; idx < tagsIndex.Max; idx++) { if (idx >= from && idx < to) { this.CompareTagsCollections(tagsIndex.Get(idx), tagsIndexReadonly.Get(idx)); } else { Assert.IsNull(tagsIndexReadonly.Get(idx)); } } from = 10; to = 1000; tagsIndexReadonly = this.SerializeDeserialize(tagsIndex, from, to); Assert.AreEqual(System.Math.Min(to, tagsIndex.Max), tagsIndexReadonly.Max); for (uint idx = 0; idx < tagsIndex.Max; idx++) { if (idx >= from && idx < to) { this.CompareTagsCollections(tagsIndex.Get(idx), tagsIndexReadonly.Get(idx)); } else { Assert.IsNull(tagsIndexReadonly.Get(idx)); } } }
/// <summary> /// Serializes the tags between the given two indexes. /// </summary> /// <param name="stream"></param> /// <param name="tagsIndex"></param> /// <param name="from"></param> /// <param name="to"></param> public static void Serialize(Stream stream, ITagsCollectionIndexReadonly tagsIndex, uint from, uint to) { int begin = (int)stream.Position; // limit to tagsIndex count. if (tagsIndex.Max < to) { to = tagsIndex.Max; } // build a string index. ObjectTable<string> stringTable = new ObjectTable<string>(false); // convert tag collections to simpler objects. List<KeyValuePair<uint, List<KeyValuePair<uint, uint>>>> tagsIndexList = new List<KeyValuePair<uint, List<KeyValuePair<uint, uint>>>>(); for (uint tagId = from; tagId < to; tagId++) { TagsCollectionBase tagsCollection = tagsIndex.Get(tagId); if (tagsCollection != null) { // convert the tags collection to a list and add to the tag index. List<KeyValuePair<uint, uint>> tagsList = new List<KeyValuePair<uint, uint>>(); foreach (Tag tag in tagsCollection) { uint keyId = stringTable.Add(tag.Key); uint valueId = stringTable.Add(tag.Value); tagsList.Add(new KeyValuePair<uint, uint>( keyId, valueId)); } tagsIndexList.Add(new KeyValuePair<uint, List<KeyValuePair<uint, uint>>>(tagId, tagsList)); } } // do the serialization. TagIndexSerializer.Serialize(begin, stream, tagsIndexList, stringTable); // clear everything. tagsIndexList.Clear(); }
/// <summary> /// Serializes the tags into different indexed blocks of given size. /// </summary> /// <param name="stream"></param> /// <param name="tagsIndex"></param> /// <param name="blockSize"></param> public static void SerializeBlocks(Stream stream, ITagsCollectionIndexReadonly tagsIndex, uint blockSize) { int begin = (int)stream.Position; // calculate the amount of blocks. uint blocks = (uint)System.Math.Ceiling((float)tagsIndex.Max / (float)blockSize); // store block count. stream.Write(BitConverter.GetBytes((int)blocks), 0, 4); // store block size. stream.Write(BitConverter.GetBytes((int)blockSize), 0, 4); // move the stream to make room for the index. stream.Seek((blocks + 2) * 4 + begin, SeekOrigin.Begin); int beginBlocks = (int)stream.Position; // keep looping over these blocks. int[] blockPositions = new int[blocks]; for (uint blockIdx = 0; blockIdx < blocks; blockIdx++) { uint from = blockIdx * blockSize; uint to = from + blockSize; MemoryStream memoryStream = new MemoryStream(); TagIndexSerializer.Serialize(memoryStream, tagsIndex, from, to); byte[] compressed = Ionic.Zlib.GZipStream.CompressBuffer(memoryStream.ToArray()); memoryStream.Dispose(); stream.Write(compressed, 0, compressed.Length); blockPositions[blockIdx] = (int)stream.Position - beginBlocks; } // write the block positions. stream.Seek(begin + 8, SeekOrigin.Begin); for(int blockIdx = 0; blockIdx < blocks; blockIdx++) { stream.Write(BitConverter.GetBytes(blockPositions[blockIdx]), 0, 4); } }