/// <summary> /// Does the actual serialization of the given data structures. /// </summary> private static void Serialize(int begin, Stream stream, List <KeyValuePair <uint, List <KeyValuePair <uint, uint> > > > tagIndex, ObjectTable <string> stringTable) { RuntimeTypeModel typeModel = TagIndexSerializer.CreateTypeModel(); // move until after the index (index contains two int's, startoftagindex, endoffile). stream.Seek(begin + 8, SeekOrigin.Begin); // serialize string table. List <string> strings = new List <string>(); for (uint id = 0; id < stringTable.Count; id++) { strings.Add(stringTable.Get(id)); } stringTable.Clear(); stringTable = null; typeModel.Serialize(stream, strings); long startOfTagsIndex = stream.Position - begin; // serialize tagindex. typeModel.Serialize(stream, tagIndex); long endOfFile = stream.Position - begin; // write index. stream.Seek(begin, SeekOrigin.Begin); stream.Write(BitConverter.GetBytes((int)startOfTagsIndex), 0, 4); // write start position of tagindex. stream.Write(BitConverter.GetBytes((int)endOfFile), 0, 4); // write size of complete file. stream.Seek(begin + endOfFile, SeekOrigin.Begin); }
/// <summary> /// Serializes all the tags in the given index. This serialization preserves the id's of each tag collection. /// </summary> /// <param name="stream">The target stream.</param> /// <param name="tagsIndex">The tags index to serialize.</param> public static void Serialize(Stream stream, ITagsCollectionIndex tagsIndex) { int begin = (int)stream.Position; // build a string index. ObjectTable <string> stringTable = new ObjectTable <string>(false); // convert tag collections to simpler objects. List <KeyValuePair <uint, List <KeyValuePair <uint, uint> > > > tagsIndexList = new List <KeyValuePair <uint, List <KeyValuePair <uint, uint> > > >(); for (uint tagId = 0; tagId < tagsIndex.Max; tagId++) { TagsCollection tagsCollection = tagsIndex.Get(tagId); if (tagsCollection != null) { // convert the tags collection to a list and add to the tag index. List <KeyValuePair <uint, uint> > tagsList = new List <KeyValuePair <uint, uint> >(); foreach (Tag tag in tagsCollection) { uint keyId = stringTable.Add(tag.Key); uint valueId = stringTable.Add(tag.Value); tagsList.Add(new KeyValuePair <uint, uint>( keyId, valueId)); } tagsIndexList.Add(new KeyValuePair <uint, List <KeyValuePair <uint, uint> > >(tagId, tagsList)); } } // do the serialization. TagIndexSerializer.Serialize(begin, stream, tagsIndexList, stringTable); // clear everything. tagsIndexList.Clear(); }
/// <summary> /// Serializes the tags into different indexed blocks of given size. /// </summary> /// <param name="stream"></param> /// <param name="tagsIndex"></param> /// <param name="blockSize"></param> public static void SerializeBlocks(Stream stream, ITagsIndexReadonly tagsIndex, uint blockSize) { int begin = (int)stream.Position; // calculate the amount of blocks. uint blocks = (uint)System.Math.Ceiling((float)tagsIndex.Max / (float)blockSize); // store block count. stream.Write(BitConverter.GetBytes((int)blocks), 0, 4); // store block size. stream.Write(BitConverter.GetBytes((int)blockSize), 0, 4); // move the stream to make room for the index. stream.Seek((blocks + 2) * 4 + begin, SeekOrigin.Begin); int beginBlocks = (int)stream.Position; // keep looping over these blocks. int[] blockPositions = new int[blocks]; for (uint blockIdx = 0; blockIdx < blocks; blockIdx++) { uint from = blockIdx * blockSize; uint to = from + blockSize; TagIndexSerializer.Serialize(stream, tagsIndex, from, to); blockPositions[blockIdx] = (int)stream.Position - beginBlocks; } // write the block positions. stream.Seek(begin + 8, SeekOrigin.Begin); for (int blockIdx = 0; blockIdx < blocks; blockIdx++) { stream.Write(BitConverter.GetBytes(blockPositions[blockIdx]), 0, 4); } }
/// <summary> /// Deserializes a tags index from the given stream. /// </summary> /// <param name="stream"></param> /// <returns></returns> public static ITagsCollectionIndexReadonly Deserialize(Stream stream) { byte[] intBytes = new byte[4]; stream.Read(intBytes, 0, 4); int startOfTags = BitConverter.ToInt32(intBytes, 0); stream.Read(intBytes, 0, 4); int endOfFile = BitConverter.ToInt32(intBytes, 0); RuntimeTypeModel typeModel = TagIndexSerializer.CreateTypeModel(); byte[] stringTableBytes = new byte[startOfTags - 8]; stream.Read(stringTableBytes, 0, stringTableBytes.Length); MemoryStream memoryStream = new MemoryStream(stringTableBytes); List <string> strings = typeModel.Deserialize(memoryStream, null, typeof(List <string>)) as List <string>; memoryStream.Dispose(); byte[] tagsIndexBytes = new byte[endOfFile - startOfTags]; stream.Read(tagsIndexBytes, 0, tagsIndexBytes.Length); memoryStream = new MemoryStream(tagsIndexBytes); List <KeyValuePair <uint, List <KeyValuePair <uint, uint> > > > tagsIndexTableList = typeModel.Deserialize(memoryStream, null, typeof(List <KeyValuePair <uint, List <KeyValuePair <uint, uint> > > >)) as List <KeyValuePair <uint, List <KeyValuePair <uint, uint> > > >; memoryStream.Dispose(); Dictionary <uint, SimpleTagsCollection> tagsIndexList = new Dictionary <uint, SimpleTagsCollection>(); for (int idx = 0; idx < tagsIndexTableList.Count; idx++) { KeyValuePair <uint, List <KeyValuePair <uint, uint> > > serializedTagsCollection = tagsIndexTableList[idx]; SimpleTagsCollection tagsCollection = new SimpleTagsCollection(); if (serializedTagsCollection.Value != null) { foreach (KeyValuePair <uint, uint> pair in serializedTagsCollection.Value) { tagsCollection.Add( new Tag(strings[(int)pair.Key], strings[(int)pair.Value])); } } tagsIndexList.Add(serializedTagsCollection.Key, tagsCollection); } return(new TagsIndexReadonly(tagsIndexList)); }
/// <summary> /// Deserializes a block. /// </summary> /// <param name="blockIdx"></param> private void DeserializeBlock(int blockIdx) { // calculate current bounds. _currentBlockMin = (uint)(blockIdx * _blockSize); // move stream to correct position. int blockOffset = 0; if (blockIdx > 0) { blockOffset = _blockPositions[blockIdx - 1]; } // seek stream. _stream.Seek(blockOffset + _begin, SeekOrigin.Begin); // deserialize this block. _currentBlock = TagIndexSerializer.Deserialize(_stream); }
/// <summary> /// Deserializes a block. /// </summary> /// <param name="blockIdx"></param> private void DeserializeBlock(int blockIdx) { // calculate current bounds. _currentBlockMin = (uint)(blockIdx * _blockSize); // move stream to correct position. int blockOffset = 0; if (blockIdx > 0) { blockOffset = _blockPositions[blockIdx - 1]; } // seek stream. _stream.Seek(blockOffset + _begin, SeekOrigin.Begin); // deserialize this block. Ionic.Zlib.GZipStream gzipStream = new Ionic.Zlib.GZipStream(_stream, Ionic.Zlib.CompressionMode.Decompress); _currentBlock = TagIndexSerializer.Deserialize(gzipStream); }
/// <summary> /// Serializes the tags into different indexed blocks of given size. /// </summary> /// <param name="stream"></param> /// <param name="tagsIndex"></param> /// <param name="blockSize"></param> public static void SerializeBlocks(Stream stream, ITagsCollectionIndexReadonly tagsIndex, uint blockSize) { int begin = (int)stream.Position; // calculate the amount of blocks. uint blocks = (uint)System.Math.Ceiling((float)tagsIndex.Max / (float)blockSize); // store block count. stream.Write(BitConverter.GetBytes((int)blocks), 0, 4); // store block size. stream.Write(BitConverter.GetBytes((int)blockSize), 0, 4); // move the stream to make room for the index. stream.Seek((blocks + 2) * 4 + begin, SeekOrigin.Begin); int beginBlocks = (int)stream.Position; // keep looping over these blocks. int[] blockPositions = new int[blocks]; for (uint blockIdx = 0; blockIdx < blocks; blockIdx++) { uint from = blockIdx * blockSize; uint to = from + blockSize; MemoryStream memoryStream = new MemoryStream(); TagIndexSerializer.Serialize(memoryStream, tagsIndex, from, to); byte[] compressed = Ionic.Zlib.GZipStream.CompressBuffer(memoryStream.ToArray()); memoryStream.Dispose(); stream.Write(compressed, 0, compressed.Length); blockPositions[blockIdx] = (int)stream.Position - beginBlocks; } // write the block positions. stream.Seek(begin + 8, SeekOrigin.Begin); for (int blockIdx = 0; blockIdx < blocks; blockIdx++) { stream.Write(BitConverter.GetBytes(blockPositions[blockIdx]), 0, 4); } }