public void MemoryMappedHugeArrayTest() { using (var intArray = new MemoryMappedHugeArrayUInt32(new MemoryMappedStream(new MemoryStream()), 1000, 1024)) { var intArrayRef = new uint[1000]; var randomGenerator = new RandomGenerator(66707770); // make this deterministic for (uint idx = 0; idx < 1000; idx++) { if (randomGenerator.Generate(2.0) > 1) { // add data. intArrayRef[idx] = idx; intArray[idx] = idx; } else { intArrayRef[idx] = int.MaxValue; intArray[idx] = int.MaxValue; } Assert.AreEqual(intArrayRef[idx], intArray[idx]); } for (int idx = 0; idx < 1000; idx++) { Assert.AreEqual(intArrayRef[idx], intArray[idx]); } } }
public void MemoryMappedHugeArrayArgumentTest() { using (var intArray = new MemoryMappedHugeArrayUInt32(new MemoryMappedStream(new MemoryStream()), 1000, 1024)) { Assert.AreEqual(1000, intArray.Length); Assert.Catch<ArgumentOutOfRangeException>(() => { intArray[1001] = 10; }); Assert.Catch<ArgumentOutOfRangeException>(() => { intArray[-1] = 10; }); uint value; Assert.Catch<ArgumentOutOfRangeException>(() => { value = intArray[1001]; }); Assert.Catch<ArgumentOutOfRangeException>(() => { value = intArray[-1]; }); } }
public void MemoryMappedHugeArrayArgumentTest() { using (var intArray = new MemoryMappedHugeArrayUInt32(new MemoryMappedStream(new MemoryStream()), 1000, 1024)) { Assert.AreEqual(1000, intArray.Length); Assert.Catch <ArgumentOutOfRangeException>(() => { intArray[1001] = 10; }); Assert.Catch <ArgumentOutOfRangeException>(() => { intArray[-1] = 10; }); uint value; Assert.Catch <ArgumentOutOfRangeException>(() => { value = intArray[1001]; }); Assert.Catch <ArgumentOutOfRangeException>(() => { value = intArray[-1]; }); } }
public void CompareToArrayTest() { using (var intArray = new MemoryMappedHugeArrayUInt32( new MemoryMappedStream(new MemoryStream()), 1000, 1024)) { var intArrayRef = new uint[1000]; var randomGenerator = new RandomGenerator(66707770); // make this deterministic for (uint idx = 0; idx < 1000; idx++) { if (randomGenerator.Generate(2.0) > 1) { // add data. intArrayRef[idx] = idx; intArray[idx] = idx; } else { intArrayRef[idx] = int.MaxValue; intArray[idx] = int.MaxValue; } Assert.AreEqual(intArrayRef[idx], intArray[idx]); } for (var idx = 0; idx < 1000; idx++) { Assert.AreEqual(intArrayRef[idx], intArray[idx]); } } }
public void ZeroSizeTest() { using (var array = new MemoryMappedHugeArrayUInt32( new MemoryMappedStream(new MemoryStream()), 0, 1024)) { Assert.AreEqual(0, array.Length); } using (var array = new MemoryMappedHugeArrayUInt32( new MemoryMappedStream(new MemoryStream()), 100, 1024)) { array.Resize(0); Assert.AreEqual(0, array.Length); } }
public void MemoryMappedHugeArrayResizeTests() { var randomGenerator = new RandomGenerator(66707770); // make this deterministic using (var intArray = new MemoryMappedHugeArrayUInt32(new MemoryMappedStream(new MemoryStream()), 1000, 256)) { var intArrayRef = new uint[1000]; for (uint idx = 0; idx < 1000; idx++) { if (randomGenerator.Generate(2.0) > 1) { // add data. intArrayRef[idx] = idx; intArray[idx] = idx; } else { intArrayRef[idx] = int.MaxValue; intArray[idx] = int.MaxValue; } Assert.AreEqual(intArrayRef[idx], intArray[idx]); } Array.Resize <uint>(ref intArrayRef, 335); intArray.Resize(335); Assert.AreEqual(intArrayRef.Length, intArray.Length); for (int idx = 0; idx < intArrayRef.Length; idx++) { Assert.AreEqual(intArrayRef[idx], intArray[idx]); } } using (var intArray = new MemoryMappedHugeArrayUInt32(new MemoryMappedStream(new MemoryStream()), 1000, 256)) { var intArrayRef = new uint[1000]; for (uint idx = 0; idx < 1000; idx++) { if (randomGenerator.Generate(2.0) > 1) { // add data. intArrayRef[idx] = idx; intArray[idx] = idx; } else { intArrayRef[idx] = int.MaxValue; intArray[idx] = int.MaxValue; } Assert.AreEqual(intArrayRef[idx], intArray[idx]); } Array.Resize <uint>(ref intArrayRef, 1235); var oldSize = intArray.Length; intArray.Resize(1235); // huge array is not initialized. for (long idx = oldSize; idx < intArray.Length; idx++) { intArray[idx] = 0; } Assert.AreEqual(intArrayRef.Length, intArray.Length); for (int idx = 0; idx < intArrayRef.Length; idx++) { Assert.AreEqual(intArrayRef[idx], intArray[idx], string.Format("Array element not equal at index: {0}. Expected {1}, found {2}", idx, intArray[idx], intArrayRef[idx])); } } }
/// <summary> /// Deserializes a graph from the given stream. /// </summary> /// <param name="stream">The stream to read from. Reading will start at position 0.</param> /// <param name="edgeDataSize">The edge data size.</param> /// <param name="mapFrom">The map from for the edge data.</param> /// <param name="mapTo">The map to for the edge data.</param> /// <param name="copy">Flag to make an in-memory copy.</param> /// <returns></returns> public new static DirectedGraph <TEdgeData> Deserialize(System.IO.Stream stream, int edgeDataSize, MappedHugeArray <TEdgeData, uint> .MapFrom mapFrom, MappedHugeArray <TEdgeData, uint> .MapTo mapTo, bool copy) { // read sizes. long position = 0; stream.Seek(4, System.IO.SeekOrigin.Begin); position = position + 4; var longBytes = new byte[8]; stream.Read(longBytes, 0, 8); position = position + 8; var vertexLength = BitConverter.ToInt64(longBytes, 0); stream.Read(longBytes, 0, 8); position = position + 8; var edgeLength = BitConverter.ToInt64(longBytes, 0); var bufferSize = 32; var cacheSize = MemoryMappedHugeArrayUInt32.DefaultCacheSize; var file = new MemoryMappedStream(new OsmSharp.IO.LimitedStream(stream)); var vertexArray = new MemoryMappedHugeArrayUInt32(file, (vertexLength + 1) * VERTEX_SIZE, (vertexLength + 1) * VERTEX_SIZE, bufferSize / 4, cacheSize * 4); position = position + ((vertexLength + 1) * VERTEX_SIZE * 4); var vertexCoordinateArray = new MappedHugeArray <GeoCoordinateSimple, float>( new MemoryMappedHugeArraySingle(file, (vertexLength + 1) * 2, (vertexLength + 1) * 2, bufferSize / 4, cacheSize * 4), 2, (array, idx, coordinate) => { array[idx] = coordinate.Latitude; array[idx + 1] = coordinate.Longitude; }, (Array, idx) => { return(new GeoCoordinateSimple() { Latitude = Array[idx], Longitude = Array[idx + 1] }); }); position = position + ((vertexLength + 1) * 2 * 4); var edgeArray = new MemoryMappedHugeArrayUInt32(file, edgeLength * EDGE_SIZE, edgeLength * EDGE_SIZE, bufferSize / 2, cacheSize * 4); position = position + (edgeLength * EDGE_SIZE * 4); var edgeDataArray = new MappedHugeArray <TEdgeData, uint>( new MemoryMappedHugeArrayUInt32(file, edgeLength * edgeDataSize, edgeLength * edgeDataSize, bufferSize * 2, cacheSize * 2), edgeDataSize, mapTo, mapFrom); position = position + (edgeLength * edgeDataSize * 4); // deserialize shapes. stream.Seek(position, System.IO.SeekOrigin.Begin); var cappedStream = new OsmSharp.IO.LimitedStream(stream); var shapes = HugeCoordinateCollectionIndex.Deserialize(cappedStream, copy); if (copy) { // copy the data. var vertexArrayCopy = new HugeArray <uint>(vertexArray.Length); vertexArrayCopy.CopyFrom(vertexArray); var vertexCoordinateArrayCopy = new HugeArray <GeoCoordinateSimple>(vertexCoordinateArray.Length); vertexCoordinateArrayCopy.CopyFrom(vertexCoordinateArray); var edgeArrayCopy = new HugeArray <uint>(edgeArray.Length); edgeArrayCopy.CopyFrom(edgeArray); var edgeDataArrayCopy = new HugeArray <TEdgeData>(edgeDataArray.Length); edgeDataArrayCopy.CopyFrom(edgeDataArray); file.Dispose(); return(new DirectedGraph <TEdgeData>(vertexCoordinateArrayCopy, vertexArrayCopy, edgeArrayCopy, edgeDataArrayCopy, shapes)); } return(new DirectedGraph <TEdgeData>(vertexCoordinateArray, vertexArray, edgeArray, edgeDataArray, shapes)); }
/// <summary> /// Serializes this graph to disk. /// </summary> /// <param name="stream">The stream to write to. Writing will start at position 0.</param> /// <param name="edgeDataSize">The edge data size.</param> /// <param name="mapFrom">The map from for the edge data.</param> /// <param name="mapTo">The map to for the edge data.</param> public override long Serialize(System.IO.Stream stream, int edgeDataSize, MappedHugeArray <TEdgeData, uint> .MapFrom mapFrom, MappedHugeArray <TEdgeData, uint> .MapTo mapTo) { long vertexCount = (_nextVertexId - 1); long edgeCount = (_nextEdgeId / EDGE_SIZE); // write vertex and edge count. long position = 0; stream.Write(BitConverter.GetBytes((int)1), 0, 4); position = position + 4; stream.Write(BitConverter.GetBytes(vertexCount), 0, 8); // write exact number of vertices. position = position + 8; stream.Write(BitConverter.GetBytes(edgeCount), 0, 8); // write exact number of edges. position = position + 8; // write in this order: vertices, vertexCoordinates, edges, edgeData, edgeShapes. using (var file = new MemoryMappedStream(new OsmSharp.IO.LimitedStream(stream))) { // write vertices (each vertex = 1 uint (4 bytes)). var vertexArray = new MemoryMappedHugeArrayUInt32(file, (vertexCount + 1) * VERTEX_SIZE, (vertexCount + 1) * VERTEX_SIZE, 1024); vertexArray.CopyFrom(_vertices, 0, 0, (vertexCount + 1) * VERTEX_SIZE); vertexArray.Dispose(); // written, get rid of it! position = position + ((vertexCount + 1) * VERTEX_SIZE * 4); // write vertex coordinates (each vertex coordinate = 2 floats (8 bytes)). var vertexCoordinateArray = new MappedHugeArray <GeoCoordinateSimple, float>( new MemoryMappedHugeArraySingle(file, (vertexCount + 1) * 2, (vertexCount + 1) * 2, 1024), 2, (array, idx, coordinate) => { array[idx] = coordinate.Latitude; array[idx + 1] = coordinate.Longitude; }, (Array, idx) => { return(new GeoCoordinateSimple() { Latitude = Array[idx], Longitude = Array[idx + 1] }); }); vertexCoordinateArray.CopyFrom(_coordinates, 0, 0, (vertexCount + 1)); vertexCoordinateArray.Dispose(); // written, get rid of it! position = position + ((vertexCount + 1) * 2 * 4); // write edges (each edge = 4 uints (16 bytes)). var edgeArray = new MemoryMappedHugeArrayUInt32(file, edgeCount * EDGE_SIZE, edgeCount * EDGE_SIZE, 1024); edgeArray.CopyFrom(_edges, edgeCount * EDGE_SIZE); edgeArray.Dispose(); // written, get rid of it! position = position + (edgeCount * EDGE_SIZE * 4); // write edge data (each edgeData = edgeDataSize units (edgeDataSize * 4 bytes)). var edgeDataArray = new MappedHugeArray <TEdgeData, uint>( new MemoryMappedHugeArrayUInt32(file, edgeCount * edgeDataSize, edgeCount * edgeDataSize, 1024), edgeDataSize, mapTo, mapFrom); edgeDataArray.CopyFrom(_edgeData, edgeCount); edgeDataArray.Dispose(); // written, get rid of it! position = position + (edgeCount * edgeDataSize * 4); } // write shapes. stream.Seek(position, System.IO.SeekOrigin.Begin); position = position + _edgeShapes.Serialize(new OsmSharp.IO.LimitedStream(stream)); return(position); }
public void MemoryMappedHugeArrayResizeTests() { var randomGenerator = new RandomGenerator(66707770); // make this deterministic using (var intArray = new MemoryMappedHugeArrayUInt32(new MemoryMappedStream(new MemoryStream()), 1000, 256)) { var intArrayRef = new uint[1000]; for (uint idx = 0; idx < 1000; idx++) { if (randomGenerator.Generate(2.0) > 1) { // add data. intArrayRef[idx] = idx; intArray[idx] = idx; } else { intArrayRef[idx] = int.MaxValue; intArray[idx] = int.MaxValue; } Assert.AreEqual(intArrayRef[idx], intArray[idx]); } Array.Resize<uint>(ref intArrayRef, 335); intArray.Resize(335); Assert.AreEqual(intArrayRef.Length, intArray.Length); for (int idx = 0; idx < intArrayRef.Length; idx++) { Assert.AreEqual(intArrayRef[idx], intArray[idx]); } } using (var intArray = new MemoryMappedHugeArrayUInt32(new MemoryMappedStream(new MemoryStream()), 1000, 256)) { var intArrayRef = new uint[1000]; for (uint idx = 0; idx < 1000; idx++) { if (randomGenerator.Generate(2.0) > 1) { // add data. intArrayRef[idx] = idx; intArray[idx] = idx; } else { intArrayRef[idx] = int.MaxValue; intArray[idx] = int.MaxValue; } Assert.AreEqual(intArrayRef[idx], intArray[idx]); } Array.Resize<uint>(ref intArrayRef, 1235); var oldSize = intArray.Length; intArray.Resize(1235); // huge array is not initialized. for (long idx = oldSize; idx < intArray.Length;idx++) { intArray[idx] = 0; } Assert.AreEqual(intArrayRef.Length, intArray.Length); for (int idx = 0; idx < intArrayRef.Length; idx++) { Assert.AreEqual(intArrayRef[idx], intArray[idx], string.Format("Array element not equal at index: {0}. Expected {1}, found {2}", idx, intArray[idx], intArrayRef[idx])); } } }