Example #1
0
        /// <summary>
        /// Does the v2 deserialization.
        /// </summary>
        /// <param name="stream"></param>
        /// <param name="lazy"></param>
        /// <param name="vehicles"></param>
        /// <returns></returns>
        protected override IBasicRouterDataSource <CHEdgeData> DoDeserialize(
            LimitedStream stream, bool lazy, IEnumerable <string> vehicles)
        {
            var intBytes = new byte[4];

            stream.Read(intBytes, 0, 4);
            int startOfBlocks = BitConverter.ToInt32(intBytes, 0);

            stream.Read(intBytes, 0, 4);
            int startOfTags = BitConverter.ToInt32(intBytes, 0);

            stream.Read(intBytes, 0, 4);
            int sizeRegionIndex = BitConverter.ToInt32(intBytes, 0);

            // deserialize regions index.
            var chVertexRegionIndex = (CHVertexRegionIndex)_runtimeTypeModel.Deserialize(
                new CappedStream(stream, stream.Position, sizeRegionIndex), null,
                typeof(CHVertexRegionIndex));

            // deserialize blocks index.
            stream.Seek(startOfBlocks, SeekOrigin.Begin);
            stream.Read(intBytes, 0, 4);
            int sizeBlockIndex = BitConverter.ToInt32(intBytes, 0);
            var chBlockIndex   = (CHBlockIndex)_runtimeTypeModel.Deserialize(
                new CappedStream(stream, stream.Position, sizeBlockIndex), null,
                typeof(CHBlockIndex));

            // deserialize tags.
            stream.Seek(startOfTags, SeekOrigin.Begin);
            ITagsCollectionIndexReadonly tagsIndex = TagIndexSerializer.DeserializeBlocks(stream);

            return(new v2.CHEdgeDataDataSource(stream, this, vehicles, sizeRegionIndex + 12,
                                               chVertexRegionIndex, _regionZoom, startOfBlocks + sizeBlockIndex + 4, chBlockIndex, (uint)_blockVertexSize,
                                               tagsIndex));
        }
        /// <summary>
        /// Creates a new osm memory router data source.
        /// </summary>
        /// <param name="graph"></param>
        /// <param name="tagsIndex"></param>
        /// <exception cref="ArgumentNullException"></exception>
        public DynamicGraphRouterDataSource(IDynamicGraph <TEdgeData> graph, ITagsCollectionIndexReadonly tagsIndex)
        {
            if (graph == null)
            {
                throw new ArgumentNullException("graph");
            }
            if (tagsIndex == null)
            {
                throw new ArgumentNullException("tagsIndex");
            }

            _graph       = graph;
            _vertexIndex = new QuadTree <GeoCoordinate, uint>();
            _tagsIndex   = tagsIndex;

            _supportedVehicles = new HashSet <Vehicle>();

            // add the current graph's vertices to the vertex index.
            for (uint newVertexId = 1; newVertexId < graph.VertexCount + 1; newVertexId++)
            {
                // add to the CHRegions.
                float latitude, longitude;
                graph.GetVertex(newVertexId, out latitude, out longitude);
                _vertexIndex.Add(new GeoCoordinate(latitude, longitude), newVertexId);
            }
        }
Example #3
0
        public void TestRandomBlockTagSerializatonNonBeginPositionLimitedStreamRegression()
        {
            SimpleTagsCollectionIndex tagsIndex = new SimpleTagsCollectionIndex();

            SimpleTagsCollection tagsCollection = new SimpleTagsCollection();

            for (int i = 0; i < 101; i++)
            {
                int tagCollectionSize = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10) + 1;
                for (int idx = 0; idx < tagCollectionSize; idx++)
                {
                    int tagValue = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10);
                    tagsCollection.Add(
                        string.Format("key_{0}", tagValue),
                        string.Format("value_{0}", tagValue));
                }
                uint tagsId = tagsIndex.Add(tagsCollection);
            }

            ITagsCollectionIndexReadonly tagsIndexReadonly = this.SerializeDeserializeBlockLimitedStream(tagsIndex, 10, 123);

            Assert.AreEqual(tagsIndex.Max, tagsIndexReadonly.Max);
            for (uint idx = 0; idx < tagsIndex.Max; idx++)
            {
                this.CompareTagsCollections(tagsIndex.Get(idx),
                                            tagsIndexReadonly.Get(idx));
            }
        }
        /// <summary>
        /// Executes the tests.
        /// </summary>
        public static void Test()
        {
            TagsTableCollectionIndex index = new TagsTableCollectionIndex();

            // first fill the index.
            ITagCollectionIndexTests.FillIndex(index, 100000);

            // serialize the index.
            FileInfo testFile = new FileInfo(@"test.file");

            testFile.Delete();
            Stream writeStream = testFile.OpenWrite();

            OsmSharp.Logging.Log.TraceEvent("Blocked", OsmSharp.Logging.TraceEventType.Information,
                                            "Serializing blocked file....");
            TagIndexSerializer.SerializeBlocks(writeStream, index, 100);
            writeStream.Flush();
            writeStream.Dispose();

            OsmSharp.Logging.Log.TraceEvent("Blocked", OsmSharp.Logging.TraceEventType.Information,
                                            string.Format("Serialized file: {0}KB", testFile.Length / 1024));


            // deserialize the index.
            Stream readStream = testFile.OpenRead();
            ITagsCollectionIndexReadonly readOnlyIndex = TagIndexSerializer.DeserializeBlocks(readStream);

            // test access.
            OsmSharp.Logging.Log.TraceEvent("Blocked", OsmSharp.Logging.TraceEventType.Information,
                                            "Started testing random access....");
            ITagCollectionIndexTests.TestRandomAccess("Blocked", readOnlyIndex, 1000);

            readStream.Dispose();
            testFile.Delete();
        }
Example #5
0
        public void TestRandomBlockTagSerializaton()
        {
            TagsTableCollectionIndex tagsIndex = new TagsTableCollectionIndex();

            TagsCollection tagsCollection = new TagsCollection();

            for (int i = 0; i < 100; i++)
            {
                int tagCollectionSize = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10) + 1;
                for (int idx = 0; idx < tagCollectionSize; idx++)
                {
                    int tagValue = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10);
                    tagsCollection.Add(
                        string.Format("key_{0}", tagValue),
                        string.Format("value_{0}", tagValue));
                }
                uint tagsId = tagsIndex.Add(tagsCollection);
            }

            ITagsCollectionIndexReadonly tagsIndexReadonly = this.SerializeDeserializeBlock(tagsIndex, 10, 0);

            Assert.AreEqual(tagsIndex.Max, tagsIndexReadonly.Max);
            for (uint idx = 0; idx < tagsIndex.Max; idx++)
            {
                ComparisonHelpers.CompareTags(tagsIndex.Get(idx),
                                              tagsIndexReadonly.Get(idx));
            }
        }
Example #6
0
        /// <summary>
        /// Creates a new osm memory router data source.
        /// </summary>
        /// <exception cref="ArgumentNullException"></exception>
        public DynamicGraphRouterDataSource(ITagsCollectionIndexReadonly tagsIndex, int initSize)
        {
            if (tagsIndex == null)
            {
                throw new ArgumentNullException("tagsIndex");
            }

            _graph       = new MemoryDynamicGraph <TEdgeData>(initSize);
            _vertexIndex = null; // do not create an index initially.
            _tagsIndex   = tagsIndex;

            _supportedVehicles = new HashSet <Vehicle>();
        }
        /// <summary>
        /// Creates a new osm memory router data source.
        /// </summary>
        /// <exception cref="ArgumentNullException"></exception>
        public DynamicGraphRouterDataSource(ITagsCollectionIndexReadonly tagsIndex)
        {
            if (tagsIndex == null)
            {
                throw new ArgumentNullException("tagsIndex");
            }

            _graph       = new MemoryDynamicGraph <TEdgeData>();
            _vertexIndex = new QuadTree <GeoCoordinate, uint>();
            _tagsIndex   = tagsIndex;

            _supportedVehicles = new HashSet <Vehicle>();
        }
Example #8
0
        /// <summary>
        /// Creates a new CH edge data source.
        /// </summary>
        public CHEdgeDataDataSource(Stream stream, CHEdgeDataDataSourceSerializer serializer,
                                    int startOfRegions, CHVertexRegionIndex regionIndex, int zoom,
                                    int startOfBlocks, CHBlockIndex blockIndex, uint blockSize,
                                    ITagsCollectionIndexReadonly tagsIndex)
        {
            _stream     = stream;
            _serializer = serializer;

            this.InitializeRegions(startOfRegions, regionIndex, zoom);
            this.InitializeBlocks(startOfBlocks, blockIndex, blockSize);

            _blocks    = new LRUCache <uint, CHBlock>(1000);
            _regions   = new LRUCache <ulong, CHVertexRegion>(1000);
            _tagsIndex = tagsIndex;
        }
Example #9
0
        /// <summary>
        /// Tests random access to an already filled index.
        /// </summary>
        /// <param name="name"></param>
        /// <param name="index"></param>
        /// <param name="testCount"></param>
        public static void TestRandomAccess(string name, ITagsCollectionIndexReadonly index,
                                            int testCount)
        {
            PerformanceInfoConsumer performanceInfo = new PerformanceInfoConsumer(string.Format("{0}.Access", name));

            performanceInfo.Start();
            performanceInfo.Report("Random accessing tags index {0} times...", testCount);

            for (int idx = 0; idx < testCount; idx++)
            {
                uint collectionIndex          = (uint)OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(index.Max);
                TagsCollectionBase collection = index.Get(collectionIndex);
            }

            performanceInfo.Stop();
        }
Example #10
0
        public void TestSimpleTagSerializatonLimitedStreamRegression()
        {
            TagsTableCollectionIndex tagsIndex = new TagsTableCollectionIndex();

            TagsCollection tagsCollection = new TagsCollection();

            tagsCollection.Add("key1", "value1");

            uint tagsId = tagsIndex.Add(tagsCollection);

            ITagsCollectionIndexReadonly tagsIndexReadonly = this.SerializeDeserializeLimitedStream(tagsIndex, 132);

            Assert.AreEqual(tagsIndex.Max, tagsIndexReadonly.Max);
            for (uint idx = 0; idx < tagsIndex.Max; idx++)
            {
                ComparisonHelpers.CompareTags(tagsIndex.Get(idx),
                                              tagsIndexReadonly.Get(idx));
            }
        }
Example #11
0
        public void TestSimpleTagSerializatonNonBeginPosition()
        {
            SimpleTagsCollectionIndex tagsIndex = new SimpleTagsCollectionIndex();

            SimpleTagsCollection tagsCollection = new SimpleTagsCollection();

            tagsCollection.Add("key1", "value1");

            uint tagsId = tagsIndex.Add(tagsCollection);

            ITagsCollectionIndexReadonly tagsIndexReadonly = this.SerializeDeserialize(tagsIndex, 1201);

            Assert.AreEqual(tagsIndex.Max, tagsIndexReadonly.Max);
            for (uint idx = 0; idx < tagsIndex.Max; idx++)
            {
                this.CompareTagsCollections(tagsIndex.Get(idx),
                                            tagsIndexReadonly.Get(idx));
            }
        }
Example #12
0
        /// <summary>
        /// Serializes the meta-data.
        /// </summary>
        /// <param name="stream"></param>
        /// <param name="tagsCollectionIndex"></param>
        protected virtual void SerializeTags(LimitedStream stream, ITagsCollectionIndexReadonly tagsCollectionIndex)
        {
            RuntimeTypeModel typeModel = RuntimeTypeModel.Create();

            typeModel.Add(typeof(SerializableTag), true);

            // write tags collection-count.
            var countBytes = BitConverter.GetBytes(tagsCollectionIndex.Max);

            stream.Write(countBytes, 0, 4);

            int blockSize = 1000;
            var tagsQueue = new List <SerializableTag>();

            // serialize tags collections one-by-one.
            for (uint idx = 0; idx < tagsCollectionIndex.Max; idx++)
            { // serialize objects one-by-one.
                var tagsCollection = tagsCollectionIndex.Get(idx);
                foreach (var tag in tagsCollection)
                {
                    tagsQueue.Add(new SerializableTag()
                    {
                        CollectionId = idx,
                        Key          = tag.Key,
                        Value        = tag.Value
                    });
                    if (tagsQueue.Count == blockSize)
                    { // time to serialize.
                        typeModel.SerializeWithSize(stream, tagsQueue.ToArray());
                        tagsQueue.Clear();
                    }
                }
            }

            tagsQueue.Add(new SerializableTag()
            {
                CollectionId = int.MaxValue,
                Key          = string.Empty,
                Value        = string.Empty
            });
            typeModel.SerializeWithSize(stream, tagsQueue.ToArray());
        }
Example #13
0
            /// <summary>
            /// Deserializes a block.
            /// </summary>
            /// <param name="blockIdx"></param>
            private void DeserializeBlock(int blockIdx)
            {
                // calculate current bounds.
                _currentBlockMin = (uint)(blockIdx * _blockSize);

                // move stream to correct position.
                int blockOffset = 0;

                if (blockIdx > 0)
                {
                    blockOffset = _blockPositions[blockIdx - 1];
                }

                // seek stream.
                _stream.Seek(blockOffset + _begin, SeekOrigin.Begin);

                // deserialize this block.
                Ionic.Zlib.GZipStream gzipStream = new Ionic.Zlib.GZipStream(_stream, Ionic.Zlib.CompressionMode.Decompress);
                _currentBlock = TagIndexSerializer.Deserialize(gzipStream);
            }
Example #14
0
        /// <summary>
        /// Serializes the tags into different indexed blocks of given size.
        /// </summary>
        /// <param name="stream"></param>
        /// <param name="tagsIndex"></param>
        /// <param name="blockSize"></param>
        public static void SerializeBlocks(Stream stream, ITagsCollectionIndexReadonly tagsIndex, uint blockSize)
        {
            int begin = (int)stream.Position;

            // calculate the amount of blocks.
            uint blocks = (uint)System.Math.Ceiling((float)tagsIndex.Max / (float)blockSize);

            // store block count.
            stream.Write(BitConverter.GetBytes((int)blocks), 0, 4);

            // store block size.
            stream.Write(BitConverter.GetBytes((int)blockSize), 0, 4);

            // move the stream to make room for the index.
            stream.Seek((blocks + 2) * 4 + begin, SeekOrigin.Begin);
            int beginBlocks = (int)stream.Position;

            // keep looping over these blocks.
            int[] blockPositions = new int[blocks];
            for (uint blockIdx = 0; blockIdx < blocks; blockIdx++)
            {
                uint from = blockIdx * blockSize;
                uint to   = from + blockSize;

                MemoryStream memoryStream = new MemoryStream();
                TagIndexSerializer.Serialize(memoryStream, tagsIndex, from, to);
                byte[] compressed = Ionic.Zlib.GZipStream.CompressBuffer(memoryStream.ToArray());
                memoryStream.Dispose();

                stream.Write(compressed, 0, compressed.Length);

                blockPositions[blockIdx] = (int)stream.Position - beginBlocks;
            }

            // write the block positions.
            stream.Seek(begin + 8, SeekOrigin.Begin);
            for (int blockIdx = 0; blockIdx < blocks; blockIdx++)
            {
                stream.Write(BitConverter.GetBytes(blockPositions[blockIdx]), 0, 4);
            }
        }
Example #15
0
            /// <summary>
            /// Returns the tags collection.
            /// </summary>
            /// <param name="tagsId"></param>
            /// <returns></returns>
            public TagsCollectionBase Get(uint tagsId)
            {
                // check bounds of current block.
                if (_currentBlock != null)
                {
                    if (tagsId >= _currentBlockMin &&
                        tagsId < (_currentBlockMin + _blockSize))
                    { // tag is in current block.
                        return(_currentBlock.Get(tagsId));
                    }
                }

                // load another block.
                int blockIdx = (int)System.Math.Floor((float)tagsId / (float)_blockSize);

                // check if outside of the scope of this index.
                if (blockIdx >= _blockPositions.Length)
                {
                    return(new TagsCollection());
                }

                // check cache.
                KeyValuePair <uint, ITagsCollectionIndexReadonly> blockOut;

                if (!_cache.TryGet(blockIdx, out blockOut))
                { // cache miss!
                    // deserialize block.
                    this.DeserializeBlock(blockIdx);

                    _cache.Add(blockIdx, new KeyValuePair <uint, ITagsCollectionIndexReadonly>(
                                   _currentBlockMin, _currentBlock));
                }
                else
                { // cache hit!
                    _currentBlock    = blockOut.Value;
                    _currentBlockMin = blockOut.Key;
                }

                return(_currentBlock.Get(tagsId));
            }
Example #16
0
        /// <summary>
        /// Serializes the tags between the given two indexes.
        /// </summary>
        /// <param name="stream"></param>
        /// <param name="tagsIndex"></param>
        /// <param name="from"></param>
        /// <param name="to"></param>
        public static void Serialize(Stream stream, ITagsCollectionIndexReadonly tagsIndex, uint from, uint to)
        {
            int begin = (int)stream.Position;

            // limit to tagsIndex count.
            if (tagsIndex.Max < to)
            {
                to = tagsIndex.Max;
            }

            // build a string index.
            ObjectTable <string> stringTable = new ObjectTable <string>(false);

            // convert tag collections to simpler objects.
            List <KeyValuePair <uint, List <KeyValuePair <uint, uint> > > > tagsIndexList = new List <KeyValuePair <uint, List <KeyValuePair <uint, uint> > > >();

            for (uint tagId = from; tagId < to; tagId++)
            {
                TagsCollectionBase tagsCollection = tagsIndex.Get(tagId);
                if (tagsCollection != null)
                { // convert the tags collection to a list and add to the tag index.
                    List <KeyValuePair <uint, uint> > tagsList = new List <KeyValuePair <uint, uint> >();
                    foreach (Tag tag in tagsCollection)
                    {
                        uint keyId   = stringTable.Add(tag.Key);
                        uint valueId = stringTable.Add(tag.Value);

                        tagsList.Add(new KeyValuePair <uint, uint>(
                                         keyId, valueId));
                    }
                    tagsIndexList.Add(new KeyValuePair <uint, List <KeyValuePair <uint, uint> > >(tagId, tagsList));
                }
            }

            // do the serialization.
            TagIndexSerializer.Serialize(begin, stream, tagsIndexList, stringTable);

            // clear everything.
            tagsIndexList.Clear();
        }
        /// <summary>
        /// Creates a new CH edge data source.
        /// </summary>
        public CHEdgeDataDataSource(Stream stream, CHEdgeDataDataSourceSerializer serializer, IEnumerable <string> vehicles,
                                    int startOfRegions, CHVertexRegionIndex regionIndex, int zoom,
                                    int startOfBlocks, CHBlockIndex blockIndex, uint blockSize,
                                    int startOfShapes, CHBlockIndex shapeIndex,
                                    int startOfReverses, CHBlockIndex reversesIndex,
                                    ITagsCollectionIndexReadonly tagsIndex)
        {
            _stream     = stream;
            _serializer = serializer;
            _vehicles   = new HashSet <string>(vehicles);

            this.InitializeRegions(startOfRegions, regionIndex, zoom);
            this.InitializeBlocks(startOfBlocks, blockIndex, blockSize);
            this.InitializeShapes(startOfShapes, shapeIndex);
            this.InitializeReverses(startOfReverses, reversesIndex);

            _blocks        = new LRUCache <uint, CHBlock>(5000);
            _blockShapes   = new LRUCache <uint, CHBlockCoordinates>(1000);
            _blockReverses = new LRUCache <uint, CHBlockReverse>(1000);
            _regions       = new LRUCache <ulong, CHVertexRegion>(1000);
            _tagsIndex     = tagsIndex;
        }
        /// <summary>
        /// Tests random access to an already filled index.
        /// </summary>
        /// <param name="name"></param>
        /// <param name="index"></param>
        /// <param name="testCount"></param>
        public static void TestRandomAccess(string name, ITagsCollectionIndexReadonly index,
            int testCount)
        {
            PerformanceInfoConsumer performanceInfo = new PerformanceInfoConsumer(string.Format("{0}.Access", name));
            performanceInfo.Start();
            performanceInfo.Report("Random accessing tags index {0} times...", testCount);

            for (int idx = 0; idx < testCount; idx++)
            {
                uint collectionIndex = (uint)OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(index.Max);
                TagsCollectionBase collection = index.Get(collectionIndex);
            }

            performanceInfo.Stop();
        }
Example #19
0
        public void TestRandomPartialTagSerializaton()
        {
            SimpleTagsCollectionIndex tagsIndex = new SimpleTagsCollectionIndex();

            SimpleTagsCollection tagsCollection = new SimpleTagsCollection();

            for (int i = 0; i < 100; i++)
            {
                int tagCollectionSize = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10) + 1;
                for (int idx = 0; idx < tagCollectionSize; idx++)
                {
                    int tagValue = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10);
                    tagsCollection.Add(
                        string.Format("key_{0}", tagValue),
                        string.Format("value_{0}", tagValue));
                }
                uint tagsId = tagsIndex.Add(tagsCollection);
            }

            uint from = 40;
            uint to   = 50;

            ITagsCollectionIndexReadonly tagsIndexReadonly = this.SerializeDeserialize(tagsIndex, from, to);

            Assert.AreEqual(System.Math.Min(to, tagsIndex.Max), tagsIndexReadonly.Max);
            for (uint idx = 0; idx < tagsIndex.Max; idx++)
            {
                if (idx >= from && idx < to)
                {
                    this.CompareTagsCollections(tagsIndex.Get(idx),
                                                tagsIndexReadonly.Get(idx));
                }
                else
                {
                    Assert.IsNull(tagsIndexReadonly.Get(idx));
                }
            }

            from = 0;
            to   = 100;

            tagsIndexReadonly = this.SerializeDeserialize(tagsIndex, from, to);
            Assert.AreEqual(System.Math.Min(to, tagsIndex.Max), tagsIndexReadonly.Max);
            for (uint idx = 0; idx < tagsIndex.Max; idx++)
            {
                if (idx >= from && idx < to)
                {
                    this.CompareTagsCollections(tagsIndex.Get(idx),
                                                tagsIndexReadonly.Get(idx));
                }
                else
                {
                    Assert.IsNull(tagsIndexReadonly.Get(idx));
                }
            }

            from = 10;
            to   = 1000;

            tagsIndexReadonly = this.SerializeDeserialize(tagsIndex, from, to);
            Assert.AreEqual(System.Math.Min(to, tagsIndex.Max), tagsIndexReadonly.Max);
            for (uint idx = 0; idx < tagsIndex.Max; idx++)
            {
                if (idx >= from && idx < to)
                {
                    this.CompareTagsCollections(tagsIndex.Get(idx),
                                                tagsIndexReadonly.Get(idx));
                }
                else
                {
                    Assert.IsNull(tagsIndexReadonly.Get(idx));
                }
            }
        }
Example #20
0
        /// <summary>
        /// Serializes the tags between the given two indexes.
        /// </summary>
        /// <param name="stream"></param>
        /// <param name="tagsIndex"></param>
        /// <param name="from"></param>
        /// <param name="to"></param>
        public static void Serialize(Stream stream, ITagsCollectionIndexReadonly tagsIndex, uint from, uint to)
        {
            int begin = (int)stream.Position;

            // limit to tagsIndex count.
            if (tagsIndex.Max < to)
            {
                to = tagsIndex.Max;
            }

            // build a string index.
            ObjectTable<string> stringTable = new ObjectTable<string>(false);

            // convert tag collections to simpler objects.
            List<KeyValuePair<uint, List<KeyValuePair<uint, uint>>>> tagsIndexList = new List<KeyValuePair<uint, List<KeyValuePair<uint, uint>>>>();
            for (uint tagId = from; tagId < to; tagId++)
            {
                TagsCollectionBase tagsCollection = tagsIndex.Get(tagId);
                if (tagsCollection != null)
                { // convert the tags collection to a list and add to the tag index.
                    List<KeyValuePair<uint, uint>> tagsList = new List<KeyValuePair<uint, uint>>();
                    foreach (Tag tag in tagsCollection)
                    {
                        uint keyId = stringTable.Add(tag.Key);
                        uint valueId = stringTable.Add(tag.Value);

                        tagsList.Add(new KeyValuePair<uint, uint>(
                            keyId, valueId));
                    }
                    tagsIndexList.Add(new KeyValuePair<uint, List<KeyValuePair<uint, uint>>>(tagId, tagsList));
                }
            }

            // do the serialization.
            TagIndexSerializer.Serialize(begin, stream, tagsIndexList, stringTable);

            // clear everything.
            tagsIndexList.Clear();
        }
Example #21
0
        /// <summary>
        /// Serializes the tags into different indexed blocks of given size.
        /// </summary>
        /// <param name="stream"></param>
        /// <param name="tagsIndex"></param>
        /// <param name="blockSize"></param>
        public static void SerializeBlocks(Stream stream, ITagsCollectionIndexReadonly tagsIndex, uint blockSize)
        {
            int begin = (int)stream.Position;

            // calculate the amount of blocks.
            uint blocks = (uint)System.Math.Ceiling((float)tagsIndex.Max / (float)blockSize);

            // store block count.
            stream.Write(BitConverter.GetBytes((int)blocks), 0, 4);

            // store block size.
            stream.Write(BitConverter.GetBytes((int)blockSize), 0, 4);

            // move the stream to make room for the index.
            stream.Seek((blocks + 2) * 4 + begin, SeekOrigin.Begin);
            int beginBlocks = (int)stream.Position;

            // keep looping over these blocks.
            int[] blockPositions = new int[blocks];
            for (uint blockIdx = 0; blockIdx < blocks; blockIdx++)
            {
                uint from = blockIdx * blockSize;
                uint to = from + blockSize;

                MemoryStream memoryStream = new MemoryStream();
                TagIndexSerializer.Serialize(memoryStream, tagsIndex, from, to);
                byte[] compressed = Ionic.Zlib.GZipStream.CompressBuffer(memoryStream.ToArray());
                memoryStream.Dispose();

                stream.Write(compressed, 0, compressed.Length);

                blockPositions[blockIdx] = (int)stream.Position - beginBlocks;
            }

            // write the block positions.
            stream.Seek(begin + 8, SeekOrigin.Begin);
            for(int blockIdx = 0; blockIdx < blocks; blockIdx++)
            {
                stream.Write(BitConverter.GetBytes(blockPositions[blockIdx]), 0, 4);
            }
        }
Example #22
0
            /// <summary>
            /// Deserializes a block.
            /// </summary>
            /// <param name="blockIdx"></param>
            private void DeserializeBlock(int blockIdx)
            {
                // calculate current bounds.
                _currentBlockMin = (uint)(blockIdx * _blockSize);

                // move stream to correct position.
                int blockOffset = 0;
                if (blockIdx > 0)
                {
                    blockOffset = _blockPositions[blockIdx - 1];
                }

                // seek stream.
                _stream.Seek(blockOffset + _begin, SeekOrigin.Begin);

                // deserialize this block.
                Ionic.Zlib.GZipStream gzipStream = new Ionic.Zlib.GZipStream(_stream, Ionic.Zlib.CompressionMode.Decompress);
                _currentBlock = TagIndexSerializer.Deserialize(gzipStream);
            }