public void MemoryMappedHugeArrayTest()
{
using (var intArray = new MemoryMappedHugeArrayUInt32(new MemoryMappedStream(new MemoryStream()), 1000, 1024))
{
var intArrayRef = new uint[1000];
var randomGenerator = new RandomGenerator(66707770); // make this deterministic
for (uint idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Generate(2.0) > 1)
{ // add data.
intArrayRef[idx] = idx;
intArray[idx] = idx;
}
else
{
intArrayRef[idx] = int.MaxValue;
intArray[idx] = int.MaxValue;
}
Assert.AreEqual(intArrayRef[idx], intArray[idx]);
}
for (int idx = 0; idx < 1000; idx++)
{
Assert.AreEqual(intArrayRef[idx], intArray[idx]);
}
}
}
public void MemoryMappedHugeArrayArgumentTest()
{
using (var intArray = new MemoryMappedHugeArrayUInt32(new MemoryMappedStream(new MemoryStream()), 1000, 1024))
{
Assert.AreEqual(1000, intArray.Length);
Assert.Catch<ArgumentOutOfRangeException>(() =>
{
intArray[1001] = 10;
});
Assert.Catch<ArgumentOutOfRangeException>(() =>
{
intArray[-1] = 10;
});
uint value;
Assert.Catch<ArgumentOutOfRangeException>(() =>
{
value = intArray[1001];
});
Assert.Catch<ArgumentOutOfRangeException>(() =>
{
value = intArray[-1];
});
}
}
public void MemoryMappedHugeArrayArgumentTest()
{
using (var intArray = new MemoryMappedHugeArrayUInt32(new MemoryMappedStream(new MemoryStream()), 1000, 1024))
{
Assert.AreEqual(1000, intArray.Length);
Assert.Catch <ArgumentOutOfRangeException>(() =>
{
intArray[1001] = 10;
});
Assert.Catch <ArgumentOutOfRangeException>(() =>
{
intArray[-1] = 10;
});
uint value;
Assert.Catch <ArgumentOutOfRangeException>(() =>
{
value = intArray[1001];
});
Assert.Catch <ArgumentOutOfRangeException>(() =>
{
value = intArray[-1];
});
}
}
public void CompareToArrayTest()
{
using (var intArray = new MemoryMappedHugeArrayUInt32(
new MemoryMappedStream(new MemoryStream()), 1000, 1024))
{
var intArrayRef = new uint[1000];
var randomGenerator = new RandomGenerator(66707770); // make this deterministic
for (uint idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Generate(2.0) > 1)
{ // add data.
intArrayRef[idx] = idx;
intArray[idx] = idx;
}
else
{
intArrayRef[idx] = int.MaxValue;
intArray[idx] = int.MaxValue;
}
Assert.AreEqual(intArrayRef[idx], intArray[idx]);
}
for (var idx = 0; idx < 1000; idx++)
{
Assert.AreEqual(intArrayRef[idx], intArray[idx]);
}
}
}
public void ZeroSizeTest()
{
using (var array = new MemoryMappedHugeArrayUInt32(
new MemoryMappedStream(new MemoryStream()), 0, 1024))
{
Assert.AreEqual(0, array.Length);
}
using (var array = new MemoryMappedHugeArrayUInt32(
new MemoryMappedStream(new MemoryStream()), 100, 1024))
{
array.Resize(0);
Assert.AreEqual(0, array.Length);
}
}
public void MemoryMappedHugeArrayResizeTests()
{
var randomGenerator = new RandomGenerator(66707770); // make this deterministic
using (var intArray = new MemoryMappedHugeArrayUInt32(new MemoryMappedStream(new MemoryStream()), 1000, 256))
{
var intArrayRef = new uint[1000];
for (uint idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Generate(2.0) > 1)
{ // add data.
intArrayRef[idx] = idx;
intArray[idx] = idx;
}
else
{
intArrayRef[idx] = int.MaxValue;
intArray[idx] = int.MaxValue;
}
Assert.AreEqual(intArrayRef[idx], intArray[idx]);
}
Array.Resize <uint>(ref intArrayRef, 335);
intArray.Resize(335);
Assert.AreEqual(intArrayRef.Length, intArray.Length);
for (int idx = 0; idx < intArrayRef.Length; idx++)
{
Assert.AreEqual(intArrayRef[idx], intArray[idx]);
}
}
using (var intArray = new MemoryMappedHugeArrayUInt32(new MemoryMappedStream(new MemoryStream()), 1000, 256))
{
var intArrayRef = new uint[1000];
for (uint idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Generate(2.0) > 1)
{ // add data.
intArrayRef[idx] = idx;
intArray[idx] = idx;
}
else
{
intArrayRef[idx] = int.MaxValue;
intArray[idx] = int.MaxValue;
}
Assert.AreEqual(intArrayRef[idx], intArray[idx]);
}
Array.Resize <uint>(ref intArrayRef, 1235);
var oldSize = intArray.Length;
intArray.Resize(1235);
// huge array is not initialized.
for (long idx = oldSize; idx < intArray.Length; idx++)
{
intArray[idx] = 0;
}
Assert.AreEqual(intArrayRef.Length, intArray.Length);
for (int idx = 0; idx < intArrayRef.Length; idx++)
{
Assert.AreEqual(intArrayRef[idx], intArray[idx], string.Format("Array element not equal at index: {0}. Expected {1}, found {2}",
idx, intArray[idx], intArrayRef[idx]));
}
}
}
/// <summary>
/// Deserializes a graph from the given stream.
/// </summary>
/// <param name="stream">The stream to read from. Reading will start at position 0.</param>
/// <param name="edgeDataSize">The edge data size.</param>
/// <param name="mapFrom">The map from for the edge data.</param>
/// <param name="mapTo">The map to for the edge data.</param>
/// <param name="copy">Flag to make an in-memory copy.</param>
/// <returns></returns>
public new static DirectedGraph <TEdgeData> Deserialize(System.IO.Stream stream, int edgeDataSize, MappedHugeArray <TEdgeData, uint> .MapFrom mapFrom,
MappedHugeArray <TEdgeData, uint> .MapTo mapTo, bool copy)
{
// read sizes.
long position = 0;
stream.Seek(4, System.IO.SeekOrigin.Begin);
position = position + 4;
var longBytes = new byte[8];
stream.Read(longBytes, 0, 8);
position = position + 8;
var vertexLength = BitConverter.ToInt64(longBytes, 0);
stream.Read(longBytes, 0, 8);
position = position + 8;
var edgeLength = BitConverter.ToInt64(longBytes, 0);
var bufferSize = 32;
var cacheSize = MemoryMappedHugeArrayUInt32.DefaultCacheSize;
var file = new MemoryMappedStream(new OsmSharp.IO.LimitedStream(stream));
var vertexArray = new MemoryMappedHugeArrayUInt32(file, (vertexLength + 1) * VERTEX_SIZE, (vertexLength + 1) * VERTEX_SIZE, bufferSize / 4, cacheSize * 4);
position = position + ((vertexLength + 1) * VERTEX_SIZE * 4);
var vertexCoordinateArray = new MappedHugeArray <GeoCoordinateSimple, float>(
new MemoryMappedHugeArraySingle(file, (vertexLength + 1) * 2, (vertexLength + 1) * 2, bufferSize / 4, cacheSize * 4),
2, (array, idx, coordinate) =>
{
array[idx] = coordinate.Latitude;
array[idx + 1] = coordinate.Longitude;
},
(Array, idx) =>
{
return(new GeoCoordinateSimple()
{
Latitude = Array[idx],
Longitude = Array[idx + 1]
});
});
position = position + ((vertexLength + 1) * 2 * 4);
var edgeArray = new MemoryMappedHugeArrayUInt32(file, edgeLength * EDGE_SIZE, edgeLength * EDGE_SIZE, bufferSize / 2, cacheSize * 4);
position = position + (edgeLength * EDGE_SIZE * 4);
var edgeDataArray = new MappedHugeArray <TEdgeData, uint>(
new MemoryMappedHugeArrayUInt32(file, edgeLength * edgeDataSize, edgeLength * edgeDataSize, bufferSize * 2, cacheSize * 2), edgeDataSize, mapTo, mapFrom);
position = position + (edgeLength * edgeDataSize * 4);
// deserialize shapes.
stream.Seek(position, System.IO.SeekOrigin.Begin);
var cappedStream = new OsmSharp.IO.LimitedStream(stream);
var shapes = HugeCoordinateCollectionIndex.Deserialize(cappedStream, copy);
if (copy)
{ // copy the data.
var vertexArrayCopy = new HugeArray <uint>(vertexArray.Length);
vertexArrayCopy.CopyFrom(vertexArray);
var vertexCoordinateArrayCopy = new HugeArray <GeoCoordinateSimple>(vertexCoordinateArray.Length);
vertexCoordinateArrayCopy.CopyFrom(vertexCoordinateArray);
var edgeArrayCopy = new HugeArray <uint>(edgeArray.Length);
edgeArrayCopy.CopyFrom(edgeArray);
var edgeDataArrayCopy = new HugeArray <TEdgeData>(edgeDataArray.Length);
edgeDataArrayCopy.CopyFrom(edgeDataArray);
file.Dispose();
return(new DirectedGraph <TEdgeData>(vertexCoordinateArrayCopy, vertexArrayCopy, edgeArrayCopy, edgeDataArrayCopy, shapes));
}
return(new DirectedGraph <TEdgeData>(vertexCoordinateArray, vertexArray, edgeArray, edgeDataArray, shapes));
}
/// <summary>
/// Serializes this graph to disk.
/// </summary>
/// <param name="stream">The stream to write to. Writing will start at position 0.</param>
/// <param name="edgeDataSize">The edge data size.</param>
/// <param name="mapFrom">The map from for the edge data.</param>
/// <param name="mapTo">The map to for the edge data.</param>
public override long Serialize(System.IO.Stream stream, int edgeDataSize, MappedHugeArray <TEdgeData, uint> .MapFrom mapFrom,
MappedHugeArray <TEdgeData, uint> .MapTo mapTo)
{
long vertexCount = (_nextVertexId - 1);
long edgeCount = (_nextEdgeId / EDGE_SIZE);
// write vertex and edge count.
long position = 0;
stream.Write(BitConverter.GetBytes((int)1), 0, 4);
position = position + 4;
stream.Write(BitConverter.GetBytes(vertexCount), 0, 8); // write exact number of vertices.
position = position + 8;
stream.Write(BitConverter.GetBytes(edgeCount), 0, 8); // write exact number of edges.
position = position + 8;
// write in this order: vertices, vertexCoordinates, edges, edgeData, edgeShapes.
using (var file = new MemoryMappedStream(new OsmSharp.IO.LimitedStream(stream)))
{
// write vertices (each vertex = 1 uint (4 bytes)).
var vertexArray = new MemoryMappedHugeArrayUInt32(file, (vertexCount + 1) * VERTEX_SIZE, (vertexCount + 1) * VERTEX_SIZE, 1024);
vertexArray.CopyFrom(_vertices, 0, 0, (vertexCount + 1) * VERTEX_SIZE);
vertexArray.Dispose(); // written, get rid of it!
position = position + ((vertexCount + 1) * VERTEX_SIZE * 4);
// write vertex coordinates (each vertex coordinate = 2 floats (8 bytes)).
var vertexCoordinateArray = new MappedHugeArray <GeoCoordinateSimple, float>(
new MemoryMappedHugeArraySingle(file, (vertexCount + 1) * 2, (vertexCount + 1) * 2, 1024),
2, (array, idx, coordinate) =>
{
array[idx] = coordinate.Latitude;
array[idx + 1] = coordinate.Longitude;
},
(Array, idx) =>
{
return(new GeoCoordinateSimple()
{
Latitude = Array[idx],
Longitude = Array[idx + 1]
});
});
vertexCoordinateArray.CopyFrom(_coordinates, 0, 0, (vertexCount + 1));
vertexCoordinateArray.Dispose(); // written, get rid of it!
position = position + ((vertexCount + 1) * 2 * 4);
// write edges (each edge = 4 uints (16 bytes)).
var edgeArray = new MemoryMappedHugeArrayUInt32(file, edgeCount * EDGE_SIZE, edgeCount * EDGE_SIZE, 1024);
edgeArray.CopyFrom(_edges, edgeCount * EDGE_SIZE);
edgeArray.Dispose(); // written, get rid of it!
position = position + (edgeCount * EDGE_SIZE * 4);
// write edge data (each edgeData = edgeDataSize units (edgeDataSize * 4 bytes)).
var edgeDataArray = new MappedHugeArray <TEdgeData, uint>(
new MemoryMappedHugeArrayUInt32(file, edgeCount * edgeDataSize, edgeCount * edgeDataSize, 1024), edgeDataSize, mapTo, mapFrom);
edgeDataArray.CopyFrom(_edgeData, edgeCount);
edgeDataArray.Dispose(); // written, get rid of it!
position = position + (edgeCount * edgeDataSize * 4);
}
// write shapes.
stream.Seek(position, System.IO.SeekOrigin.Begin);
position = position + _edgeShapes.Serialize(new OsmSharp.IO.LimitedStream(stream));
return(position);
}
public void MemoryMappedHugeArrayResizeTests()
{
var randomGenerator = new RandomGenerator(66707770); // make this deterministic
using (var intArray = new MemoryMappedHugeArrayUInt32(new MemoryMappedStream(new MemoryStream()), 1000, 256))
{
var intArrayRef = new uint[1000];
for (uint idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Generate(2.0) > 1)
{ // add data.
intArrayRef[idx] = idx;
intArray[idx] = idx;
}
else
{
intArrayRef[idx] = int.MaxValue;
intArray[idx] = int.MaxValue;
}
Assert.AreEqual(intArrayRef[idx], intArray[idx]);
}
Array.Resize<uint>(ref intArrayRef, 335);
intArray.Resize(335);
Assert.AreEqual(intArrayRef.Length, intArray.Length);
for (int idx = 0; idx < intArrayRef.Length; idx++)
{
Assert.AreEqual(intArrayRef[idx], intArray[idx]);
}
}
using (var intArray = new MemoryMappedHugeArrayUInt32(new MemoryMappedStream(new MemoryStream()), 1000, 256))
{
var intArrayRef = new uint[1000];
for (uint idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Generate(2.0) > 1)
{ // add data.
intArrayRef[idx] = idx;
intArray[idx] = idx;
}
else
{
intArrayRef[idx] = int.MaxValue;
intArray[idx] = int.MaxValue;
}
Assert.AreEqual(intArrayRef[idx], intArray[idx]);
}
Array.Resize<uint>(ref intArrayRef, 1235);
var oldSize = intArray.Length;
intArray.Resize(1235);
// huge array is not initialized.
for (long idx = oldSize; idx < intArray.Length;idx++)
{
intArray[idx] = 0;
}
Assert.AreEqual(intArrayRef.Length, intArray.Length);
for (int idx = 0; idx < intArrayRef.Length; idx++)
{
Assert.AreEqual(intArrayRef[idx], intArray[idx], string.Format("Array element not equal at index: {0}. Expected {1}, found {2}",
idx, intArray[idx], intArrayRef[idx]));
}
}
}
public void ZeroSizeTest()
{
using (var array = new MemoryMappedHugeArrayUInt32(
new MemoryMappedStream(new MemoryStream()), 0, 1024))
{
Assert.AreEqual(0, array.Length);
}
using (var array = new MemoryMappedHugeArrayUInt32(
new MemoryMappedStream(new MemoryStream()), 100, 1024))
{
array.Resize(0);
Assert.AreEqual(0, array.Length);
}
}
