public void TestRandomBlockTagSerializatonNonBeginPositionLimitedStreamRegression()
{
SimpleTagsIndex tagsIndex = new SimpleTagsIndex();
SimpleTagsCollection tagsCollection = new SimpleTagsCollection();
for (int i = 0; i < 101; i++)
{
int tagCollectionSize = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10) + 1;
for (int idx = 0; idx < tagCollectionSize; idx++)
{
int tagValue = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10);
tagsCollection.Add(
string.Format("key_{0}", tagValue),
string.Format("value_{0}", tagValue));
}
uint tagsId = tagsIndex.Add(tagsCollection);
}
ITagsIndexReadonly tagsIndexReadonly = this.SerializeDeserializeBlockLimitedStream(tagsIndex, 10, 123);
Assert.AreEqual(tagsIndex.Max, tagsIndexReadonly.Max);
for (uint idx = 0; idx < tagsIndex.Max; idx++)
{
this.CompareTagsCollections(tagsIndex.Get(idx),
tagsIndexReadonly.Get(idx));
}
}
/// <summary>
/// Creates a new osm memory router data source.
/// </summary>
/// <param name="graph"></param>
/// <param name="tagsIndex"></param>
/// <exception cref="ArgumentNullException"></exception>
public DynamicGraphRouterDataSource(IDynamicGraph <TEdgeData> graph, ITagsIndexReadonly tagsIndex)
{
if (graph == null)
{
throw new ArgumentNullException("graph");
}
if (tagsIndex == null)
{
throw new ArgumentNullException("tagsIndex");
}
_graph = graph;
_vertexIndex = new QuadTree <GeoCoordinate, uint>();
_tagsIndex = tagsIndex;
_supportedVehicles = new HashSet <Vehicle>();
// add the current graph's vertices to the vertex index.
for (uint newVertexId = 1; newVertexId < graph.VertexCount + 1; newVertexId++)
{
// add to the CHRegions.
float latitude, longitude;
graph.GetVertex(newVertexId, out latitude, out longitude);
_vertexIndex.Add(new GeoCoordinate(latitude, longitude), newVertexId);
}
}
/// <summary>
/// Serializes the tags into different indexed blocks of given size.
/// </summary>
/// <param name="stream"></param>
/// <param name="tagsIndex"></param>
/// <param name="blockSize"></param>
public static void SerializeBlocks(Stream stream, ITagsIndexReadonly tagsIndex, uint blockSize)
{
int begin = (int)stream.Position;
// calculate the amount of blocks.
uint blocks = (uint)System.Math.Ceiling((float)tagsIndex.Max / (float)blockSize);
// store block count.
stream.Write(BitConverter.GetBytes((int)blocks), 0, 4);
// store block size.
stream.Write(BitConverter.GetBytes((int)blockSize), 0, 4);
// move the stream to make room for the index.
stream.Seek((blocks + 2) * 4 + begin, SeekOrigin.Begin);
int beginBlocks = (int)stream.Position;
// keep looping over these blocks.
int[] blockPositions = new int[blocks];
for (uint blockIdx = 0; blockIdx < blocks; blockIdx++)
{
uint from = blockIdx * blockSize;
uint to = from + blockSize;
TagIndexSerializer.Serialize(stream, tagsIndex, from, to);
blockPositions[blockIdx] = (int)stream.Position - beginBlocks;
}
// write the block positions.
stream.Seek(begin + 8, SeekOrigin.Begin);
for (int blockIdx = 0; blockIdx < blocks; blockIdx++)
{
stream.Write(BitConverter.GetBytes(blockPositions[blockIdx]), 0, 4);
}
}
/// <summary>
/// Does the v2 deserialization.
/// </summary>
/// <param name="stream"></param>
/// <param name="lazy"></param>
/// <returns></returns>
protected override IBasicRouterDataSource <CHEdgeData> DoDeserialize(
LimitedStream stream, bool lazy)
{
var intBytes = new byte[4];
stream.Read(intBytes, 0, 4);
int startOfBlocks = BitConverter.ToInt32(intBytes, 0);
stream.Read(intBytes, 0, 4);
int startOfTags = BitConverter.ToInt32(intBytes, 0);
stream.Read(intBytes, 0, 4);
int sizeRegionIndex = BitConverter.ToInt32(intBytes, 0);
// deserialize regions index.
var chVertexRegionIndex = (CHVertexRegionIndex)_runtimeTypeModel.Deserialize(
new CappedStream(stream, stream.Position, sizeRegionIndex), null,
typeof(CHVertexRegionIndex));
// deserialize blocks index.
stream.Seek(startOfBlocks, SeekOrigin.Begin);
stream.Read(intBytes, 0, 4);
int sizeBlockIndex = BitConverter.ToInt32(intBytes, 0);
var chBlockIndex = (CHBlockIndex)_runtimeTypeModel.Deserialize(
new CappedStream(stream, stream.Position, sizeBlockIndex), null,
typeof(CHBlockIndex));
// deserialize tags.
stream.Seek(startOfTags, SeekOrigin.Begin);
ITagsIndexReadonly tagsIndex = TagIndexSerializer.DeserializeBlocks(stream);
return(new v2.CHEdgeDataDataSource(stream, this, sizeRegionIndex + 12,
chVertexRegionIndex, RegionZoom, startOfBlocks + sizeBlockIndex + 4, chBlockIndex, BlockVertexSize,
tagsIndex));
}
/// <summary>
/// Creates a new osm memory router data source.
/// </summary>
/// <exception cref="ArgumentNullException"></exception>
public DynamicGraphRouterDataSource(ITagsIndexReadonly tagsIndex)
{
if (tagsIndex == null)
{
throw new ArgumentNullException("tagsIndex");
}
_graph = new MemoryDynamicGraph <TEdgeData>();
_vertexIndex = new QuadTree <GeoCoordinate, uint>();
_tagsIndex = tagsIndex;
_supportedVehicles = new HashSet <Vehicle>();
}
/// <summary>
/// Creates a new CH edge data source.
/// </summary>
public CHEdgeDataDataSource(Stream stream, CHEdgeDataDataSourceSerializer serializer,
int startOfRegions, CHVertexRegionIndex regionIndex, int zoom,
int startOfBlocks, CHBlockIndex blockIndex, uint blockSize,
ITagsIndexReadonly tagsIndex)
{
_stream = stream;
_serializer = serializer;
this.InitializeRegions(startOfRegions, regionIndex, zoom);
this.InitializeBlocks(startOfBlocks, blockIndex, blockSize);
_blocks = new LRUCache <uint, CHBlock>(1000);
_regions = new LRUCache <ulong, CHVertexRegion>(1000);
_tagsIndex = tagsIndex;
}
public void TestSimpleTagSerializatonNonBeginPosition()
{
SimpleTagsIndex tagsIndex = new SimpleTagsIndex();
SimpleTagsCollection tagsCollection = new SimpleTagsCollection();
tagsCollection.Add("key1", "value1");
uint tagsId = tagsIndex.Add(tagsCollection);
ITagsIndexReadonly tagsIndexReadonly = this.SerializeDeserialize(tagsIndex, 1201);
Assert.AreEqual(tagsIndex.Max, tagsIndexReadonly.Max);
for (uint idx = 0; idx < tagsIndex.Max; idx++)
{
this.CompareTagsCollections(tagsIndex.Get(idx),
tagsIndexReadonly.Get(idx));
}
}
/// <summary>
/// Deserializes a block.
/// </summary>
/// <param name="blockIdx"></param>
private void DeserializeBlock(int blockIdx)
{
// calculate current bounds.
_currentBlockMin = (uint)(blockIdx * _blockSize);
// move stream to correct position.
int blockOffset = 0;
if (blockIdx > 0)
{
blockOffset = _blockPositions[blockIdx - 1];
}
// seek stream.
_stream.Seek(blockOffset + _begin, SeekOrigin.Begin);
// deserialize this block.
_currentBlock = TagIndexSerializer.Deserialize(_stream);
}
/// <summary>
/// Serializes the tags between the given two indexes.
/// </summary>
/// <param name="stream"></param>
/// <param name="tagsIndex"></param>
/// <param name="from"></param>
/// <param name="to"></param>
public static void Serialize(Stream stream, ITagsIndexReadonly tagsIndex, uint from, uint to)
{
int begin = (int)stream.Position;
// limit to tagsIndex count.
if (tagsIndex.Max < to)
{
to = tagsIndex.Max;
}
// build a string index.
ObjectTable <string> stringTable = new ObjectTable <string>(false);
// convert tag collections to simpler objects.
List <KeyValuePair <uint, List <KeyValuePair <uint, uint> > > > tagsIndexList = new List <KeyValuePair <uint, List <KeyValuePair <uint, uint> > > >();
for (uint tagId = from; tagId < to; tagId++)
{
TagsCollection tagsCollection = tagsIndex.Get(tagId);
if (tagsCollection != null)
{ // convert the tags collection to a list and add to the tag index.
List <KeyValuePair <uint, uint> > tagsList = new List <KeyValuePair <uint, uint> >();
foreach (Tag tag in tagsCollection)
{
uint keyId = stringTable.Add(tag.Key);
uint valueId = stringTable.Add(tag.Value);
tagsList.Add(new KeyValuePair <uint, uint>(
keyId, valueId));
}
tagsIndexList.Add(new KeyValuePair <uint, List <KeyValuePair <uint, uint> > >(tagId, tagsList));
}
}
// do the serialization.
TagIndexSerializer.Serialize(begin, stream, tagsIndexList, stringTable);
// clear everything.
tagsIndexList.Clear();
}
public void TestRandomPartialTagSerializaton()
{
SimpleTagsIndex tagsIndex = new SimpleTagsIndex();
SimpleTagsCollection tagsCollection = new SimpleTagsCollection();
for (int i = 0; i < 100; i++)
{
int tagCollectionSize = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10) + 1;
for (int idx = 0; idx < tagCollectionSize; idx++)
{
int tagValue = OsmSharp.Math.Random.StaticRandomGenerator.Get().Generate(10);
tagsCollection.Add(
string.Format("key_{0}", tagValue),
string.Format("value_{0}", tagValue));
}
uint tagsId = tagsIndex.Add(tagsCollection);
}
uint from = 40;
uint to = 50;
ITagsIndexReadonly tagsIndexReadonly = this.SerializeDeserialize(tagsIndex, from, to);
Assert.AreEqual(System.Math.Min(to, tagsIndex.Max), tagsIndexReadonly.Max);
for (uint idx = 0; idx < tagsIndex.Max; idx++)
{
if (idx >= from && idx < to)
{
this.CompareTagsCollections(tagsIndex.Get(idx),
tagsIndexReadonly.Get(idx));
}
else
{
Assert.IsNull(tagsIndexReadonly.Get(idx));
}
}
from = 0;
to = 100;
tagsIndexReadonly = this.SerializeDeserialize(tagsIndex, from, to);
Assert.AreEqual(System.Math.Min(to, tagsIndex.Max), tagsIndexReadonly.Max);
for (uint idx = 0; idx < tagsIndex.Max; idx++)
{
if (idx >= from && idx < to)
{
this.CompareTagsCollections(tagsIndex.Get(idx),
tagsIndexReadonly.Get(idx));
}
else
{
Assert.IsNull(tagsIndexReadonly.Get(idx));
}
}
from = 10;
to = 1000;
tagsIndexReadonly = this.SerializeDeserialize(tagsIndex, from, to);
Assert.AreEqual(System.Math.Min(to, tagsIndex.Max), tagsIndexReadonly.Max);
for (uint idx = 0; idx < tagsIndex.Max; idx++)
{
if (idx >= from && idx < to)
{
this.CompareTagsCollections(tagsIndex.Get(idx),
tagsIndexReadonly.Get(idx));
}
else
{
Assert.IsNull(tagsIndexReadonly.Get(idx));
}
}
}
/// <summary>
/// Deserializes a block.
/// </summary>
/// <param name="blockIdx"></param>
private void DeserializeBlock(int blockIdx)
{
// calculate current bounds.
_currentBlockMin = (uint)(blockIdx * _blockSize);
// move stream to correct position.
int blockOffset = 0;
if (blockIdx > 0)
{
blockOffset = _blockPositions[blockIdx - 1];
}
// seek stream.
_stream.Seek(blockOffset + _begin, SeekOrigin.Begin);
// deserialize this block.
_currentBlock = TagIndexSerializer.Deserialize(_stream);
}
/// <summary>
/// Serializes the tags into different indexed blocks of given size.
/// </summary>
/// <param name="stream"></param>
/// <param name="tagsIndex"></param>
/// <param name="blockSize"></param>
public static void SerializeBlocks(Stream stream, ITagsIndexReadonly tagsIndex, uint blockSize)
{
int begin = (int)stream.Position;
// calculate the amount of blocks.
uint blocks = (uint)System.Math.Ceiling((float)tagsIndex.Max / (float)blockSize);
// store block count.
stream.Write(BitConverter.GetBytes((int)blocks), 0, 4);
// store block size.
stream.Write(BitConverter.GetBytes((int)blockSize), 0, 4);
// move the stream to make room for the index.
stream.Seek((blocks + 2) * 4 + begin, SeekOrigin.Begin);
int beginBlocks = (int)stream.Position;
// keep looping over these blocks.
int[] blockPositions = new int[blocks];
for (uint blockIdx = 0; blockIdx < blocks; blockIdx++)
{
uint from = blockIdx * blockSize;
uint to = from + blockSize;
TagIndexSerializer.Serialize(stream, tagsIndex, from, to);
blockPositions[blockIdx] = (int)stream.Position - beginBlocks;
}
// write the block positions.
stream.Seek(begin + 8, SeekOrigin.Begin);
for(int blockIdx = 0; blockIdx < blocks; blockIdx++)
{
stream.Write(BitConverter.GetBytes(blockPositions[blockIdx]), 0, 4);
}
}
/// <summary>
/// Serializes the tags between the given two indexes.
/// </summary>
/// <param name="stream"></param>
/// <param name="tagsIndex"></param>
/// <param name="from"></param>
/// <param name="to"></param>
public static void Serialize(Stream stream, ITagsIndexReadonly tagsIndex, uint from, uint to)
{
int begin = (int)stream.Position;
// limit to tagsIndex count.
if (tagsIndex.Max < to)
{
to = tagsIndex.Max;
}
// build a string index.
ObjectTable<string> stringTable = new ObjectTable<string>(false);
// convert tag collections to simpler objects.
List<KeyValuePair<uint, List<KeyValuePair<uint, uint>>>> tagsIndexList = new List<KeyValuePair<uint, List<KeyValuePair<uint, uint>>>>();
for (uint tagId = from; tagId < to; tagId++)
{
TagsCollection tagsCollection = tagsIndex.Get(tagId);
if (tagsCollection != null)
{ // convert the tags collection to a list and add to the tag index.
List<KeyValuePair<uint, uint>> tagsList = new List<KeyValuePair<uint, uint>>();
foreach (Tag tag in tagsCollection)
{
uint keyId = stringTable.Add(tag.Key);
uint valueId = stringTable.Add(tag.Value);
tagsList.Add(new KeyValuePair<uint, uint>(
keyId, valueId));
}
tagsIndexList.Add(new KeyValuePair<uint, List<KeyValuePair<uint, uint>>>(tagId, tagsList));
}
}
// do the serialization.
TagIndexSerializer.Serialize(begin, stream, tagsIndexList, stringTable);
// clear everything.
tagsIndexList.Clear();
}
