/// <summary>
/// Deserializes a huge collection index from the given stream.
/// </summary>
/// <param name="stream">The source stream.</param>
/// <param name="copy">The copy flag. When true all data is copied into memory, otherwise the source-stream is used as a memorymapped file.</param>
/// <returns></returns>
public static HugeCoordinateCollectionIndex Deserialize(Stream stream, bool copy = false)
{
// read sizes.
long position = 0;
var longBytes = new byte[8];
stream.Read(longBytes, 0, 8);
position = position + 8;
var indexLength = BitConverter.ToInt64(longBytes, 0);
stream.Read(longBytes, 0, 8);
position = position + 8;
var coordinateLength = BitConverter.ToInt64(longBytes, 0);
var file = new MemoryMappedStream(new LimitedStream(stream));
var indexArray = new MemoryMappedHugeArrayUInt64(file, indexLength, indexLength, 1024);
var coordinateArray = new MemoryMappedHugeArraySingle(file, coordinateLength * 2, coordinateLength * 2, 1024);
if (copy)
{ // copy the data.
var indexArrayCopy = new HugeArray <ulong>(indexLength);
indexArrayCopy.CopyFrom(indexArray);
var coordinateArrayCopy = new HugeArray <float>(coordinateLength * 2);
coordinateArrayCopy.CopyFrom(coordinateArray);
file.Dispose();
return(new HugeCoordinateCollectionIndex(indexLength, indexArrayCopy, coordinateArrayCopy));
}
return(new HugeCoordinateCollectionIndex(indexLength, indexArray, coordinateArray));
}
/// <summary>
/// Deserializes an index from the given stream.
/// </summary>
/// <returns></returns>
public static MemoryMappedIndex <T> Deserialize(System.IO.Stream stream, MemoryMappedFile.ReadFromDelegate <T> readFrom,
MemoryMappedFile.WriteToDelegate <T> writeTo, bool copy, out long size)
{
stream.Seek(0, System.IO.SeekOrigin.Begin);
var longBytes = new byte[8];
stream.Read(longBytes, 0, 8);
size = BitConverter.ToInt64(longBytes, 0) + 8;
if (copy)
{
var bytes = (int)size - 8;
var memoryStream = new MemoryStream(bytes);
var buffer = new byte[32768];
int read;
while ((read = stream.Read(buffer, 0, System.Math.Min(buffer.Length, bytes))) > 0)
{
memoryStream.Write(buffer, 0, read);
bytes -= read;
}
memoryStream.Seek(0, SeekOrigin.Begin);
stream = memoryStream;
}
var file = new MemoryMappedStream(new LimitedStream(stream));
return(new MemoryMappedIndex <T>(file, readFrom, writeTo, size - 8));
}
/// <summary>
/// Serializes this huge collection index to the given stream.
/// </summary>
/// <param name="stream"></param>
public long Serialize(Stream stream)
{
// first trim to minimize the amount of useless data store.
this.Trim();
// start writing.
long indexSize = _index.Length;
long coordinatesCount = _coordinates.Length;
long position = 0;
stream.Write(BitConverter.GetBytes(indexSize), 0, 8); // write the actual index size.
position = position + 8;
stream.Write(BitConverter.GetBytes(coordinatesCount), 0, 8); // write the actual number of coordinates.
position = position + 8;
// write in this order: index, shapes.
using (var file = new MemoryMappedStream(new LimitedStream(stream)))
{
// write index.
var indexArray = new MemoryMappedHugeArrayUInt64(file, indexSize, indexSize, 1024);
indexArray.CopyFrom(_index, indexSize);
indexArray.Dispose();
position = position + (indexSize * 8);
// write coordinates.
var coordinatesArray = new MemoryMappedHugeArraySingle(file, coordinatesCount * 2, coordinatesCount * 2, 1024);
coordinatesArray.CopyFrom(_coordinates);
coordinatesArray.Dispose();
position = position + (coordinatesCount * 2 * 4);
}
return(position);
}
/// <summary>
/// Creates a new stream target.
/// </summary>
public CHEdgeFlatfileStreamTarget(Stream stream, ITagsIndex tagsIndex, Vehicle vehicle, MemoryMappedStream memoryMappedStream)
: base(new RouterDataSource<CHEdgeData>(new DirectedGraph<CHEdgeData>(memoryMappedStream, 1000,
CHEdgeData.MapFromDelegate, CHEdgeData.MapToDelegate, CHEdgeData.SizeUints), tagsIndex), new OsmRoutingInterpreter(), tagsIndex, vehicle)
{
_stream = stream;
_memoryMappedStream = memoryMappedStream;
}
/// <summary>
/// Creates the target.
/// </summary>
/// <returns></returns>
public static CHEdgeFlatfileStreamTarget CreateTarget(Stream stream, Vehicle vehicle, MemoryMappedStream memoryMappedStream)
{
if(memoryMappedStream == null)
{
return new CHEdgeFlatfileStreamTarget(stream, new TagsIndex(new MemoryMappedStream()), vehicle);
}
return new CHEdgeFlatfileStreamTarget(stream, new TagsIndex(memoryMappedStream), vehicle, memoryMappedStream);
}
/// <summary>
/// Creates the target.
/// </summary>
/// <returns></returns>
public static LiveEdgeFlatfileStreamTarget CreateTarget(Stream stream, MemoryMappedStream memoryMappedStream)
{
if (memoryMappedStream == null)
{
return new LiveEdgeFlatfileStreamTarget(stream, new TagsIndex(new MemoryMappedStream()));
}
return new LiveEdgeFlatfileStreamTarget(stream, new TagsIndex(memoryMappedStream));
}
/// <summary>
/// Deserializes a tags index from the given stream.
/// </summary>
/// <param name="stream">The stream to read from. Reading will start at position 0.</param>
/// <param name="readFrom">The delegate to read data from the stream.</param>
/// <returns></returns>
public static IndexBase <T> Deserialize(System.IO.Stream stream, MemoryMappedFile.ReadFromDelegate <T> readFrom)
{
var longBytes = new byte[8];
stream.Read(longBytes, 0, 8);
var size = BitConverter.ToInt64(longBytes, 0);
var file = new MemoryMappedStream(new OsmSharp.IO.LimitedStream(stream));
return(new MemoryMappedIndex <T>(file, readFrom,
(s, position, value) =>
{
throw new InvalidOperationException("This index is readonly.");
}, size));
}
/// <summary>
/// Deserializes an index from the given stream.
/// </summary>
/// <param name="stream">The stream to read from. Reading will start at position 0.</param>
/// <param name="readFrom"></param>
/// <param name="writeTo"></param>
/// <param name="copy">Flag to make an in-memory copy.</param>
/// <param name="size">The total size in bytes of the deserialized data.</param>
/// <returns></returns>
public static MemoryMappedIndex <T> Deserialize(System.IO.Stream stream, MemoryMappedFile.ReadFromDelegate <T> readFrom,
MemoryMappedFile.WriteToDelegate <T> writeTo, bool copy, out long size)
{
if (copy)
{
throw new NotSupportedException("Deserializing a memory mapped index with copy option is not supported.");
}
stream.Seek(0, System.IO.SeekOrigin.Begin);
var longBytes = new byte[8];
stream.Read(longBytes, 0, 8);
size = BitConverter.ToInt64(longBytes, 0) + 8;
var file = new MemoryMappedStream(new LimitedStream(stream));
return(new MemoryMappedIndex <T>(file, readFrom, writeTo, size - 8));
}
/// <summary>
/// Deserializes a graph from the given stream.
/// </summary>
/// <param name="stream">The stream to read from. Reading will start at position 0.</param>
/// <param name="edgeDataSize">The edge data size.</param>
/// <param name="mapFrom">The map from for the edge data.</param>
/// <param name="mapTo">The map to for the edge data.</param>
/// <param name="copy">Flag to make an in-memory copy.</param>
/// <returns></returns>
public new static DirectedGraph <TEdgeData> Deserialize(System.IO.Stream stream, int edgeDataSize, MappedHugeArray <TEdgeData, uint> .MapFrom mapFrom,
MappedHugeArray <TEdgeData, uint> .MapTo mapTo, bool copy)
{
// read sizes.
long position = 0;
stream.Seek(4, System.IO.SeekOrigin.Begin);
position = position + 4;
var longBytes = new byte[8];
stream.Read(longBytes, 0, 8);
position = position + 8;
var vertexLength = BitConverter.ToInt64(longBytes, 0);
stream.Read(longBytes, 0, 8);
position = position + 8;
var edgeLength = BitConverter.ToInt64(longBytes, 0);
var bufferSize = 32;
var cacheSize = MemoryMappedHugeArrayUInt32.DefaultCacheSize;
var file = new MemoryMappedStream(new OsmSharp.IO.LimitedStream(stream));
var vertexArray = new MemoryMappedHugeArrayUInt32(file, (vertexLength + 1) * VERTEX_SIZE, (vertexLength + 1) * VERTEX_SIZE, bufferSize / 4, cacheSize * 4);
position = position + ((vertexLength + 1) * VERTEX_SIZE * 4);
var vertexCoordinateArray = new MappedHugeArray <GeoCoordinateSimple, float>(
new MemoryMappedHugeArraySingle(file, (vertexLength + 1) * 2, (vertexLength + 1) * 2, bufferSize / 4, cacheSize * 4),
2, (array, idx, coordinate) =>
{
array[idx] = coordinate.Latitude;
array[idx + 1] = coordinate.Longitude;
},
(Array, idx) =>
{
return(new GeoCoordinateSimple()
{
Latitude = Array[idx],
Longitude = Array[idx + 1]
});
});
position = position + ((vertexLength + 1) * 2 * 4);
var edgeArray = new MemoryMappedHugeArrayUInt32(file, edgeLength * EDGE_SIZE, edgeLength * EDGE_SIZE, bufferSize / 2, cacheSize * 4);
position = position + (edgeLength * EDGE_SIZE * 4);
var edgeDataArray = new MappedHugeArray <TEdgeData, uint>(
new MemoryMappedHugeArrayUInt32(file, edgeLength * edgeDataSize, edgeLength * edgeDataSize, bufferSize * 2, cacheSize * 2), edgeDataSize, mapTo, mapFrom);
position = position + (edgeLength * edgeDataSize * 4);
// deserialize shapes.
stream.Seek(position, System.IO.SeekOrigin.Begin);
var cappedStream = new OsmSharp.IO.LimitedStream(stream);
var shapes = HugeCoordinateCollectionIndex.Deserialize(cappedStream, copy);
if (copy)
{ // copy the data.
var vertexArrayCopy = new HugeArray <uint>(vertexArray.Length);
vertexArrayCopy.CopyFrom(vertexArray);
var vertexCoordinateArrayCopy = new HugeArray <GeoCoordinateSimple>(vertexCoordinateArray.Length);
vertexCoordinateArrayCopy.CopyFrom(vertexCoordinateArray);
var edgeArrayCopy = new HugeArray <uint>(edgeArray.Length);
edgeArrayCopy.CopyFrom(edgeArray);
var edgeDataArrayCopy = new HugeArray <TEdgeData>(edgeDataArray.Length);
edgeDataArrayCopy.CopyFrom(edgeDataArray);
file.Dispose();
return(new DirectedGraph <TEdgeData>(vertexCoordinateArrayCopy, vertexArrayCopy, edgeArrayCopy, edgeDataArrayCopy, shapes));
}
return(new DirectedGraph <TEdgeData>(vertexCoordinateArray, vertexArray, edgeArray, edgeDataArray, shapes));
}
/// <summary>
/// Serializes this graph to disk.
/// </summary>
/// <param name="stream">The stream to write to. Writing will start at position 0.</param>
/// <param name="edgeDataSize">The edge data size.</param>
/// <param name="mapFrom">The map from for the edge data.</param>
/// <param name="mapTo">The map to for the edge data.</param>
public override long Serialize(System.IO.Stream stream, int edgeDataSize, MappedHugeArray <TEdgeData, uint> .MapFrom mapFrom,
MappedHugeArray <TEdgeData, uint> .MapTo mapTo)
{
long vertexCount = (_nextVertexId - 1);
long edgeCount = (_nextEdgeId / EDGE_SIZE);
// write vertex and edge count.
long position = 0;
stream.Write(BitConverter.GetBytes((int)1), 0, 4);
position = position + 4;
stream.Write(BitConverter.GetBytes(vertexCount), 0, 8); // write exact number of vertices.
position = position + 8;
stream.Write(BitConverter.GetBytes(edgeCount), 0, 8); // write exact number of edges.
position = position + 8;
// write in this order: vertices, vertexCoordinates, edges, edgeData, edgeShapes.
using (var file = new MemoryMappedStream(new OsmSharp.IO.LimitedStream(stream)))
{
// write vertices (each vertex = 1 uint (4 bytes)).
var vertexArray = new MemoryMappedHugeArrayUInt32(file, (vertexCount + 1) * VERTEX_SIZE, (vertexCount + 1) * VERTEX_SIZE, 1024);
vertexArray.CopyFrom(_vertices, 0, 0, (vertexCount + 1) * VERTEX_SIZE);
vertexArray.Dispose(); // written, get rid of it!
position = position + ((vertexCount + 1) * VERTEX_SIZE * 4);
// write vertex coordinates (each vertex coordinate = 2 floats (8 bytes)).
var vertexCoordinateArray = new MappedHugeArray <GeoCoordinateSimple, float>(
new MemoryMappedHugeArraySingle(file, (vertexCount + 1) * 2, (vertexCount + 1) * 2, 1024),
2, (array, idx, coordinate) =>
{
array[idx] = coordinate.Latitude;
array[idx + 1] = coordinate.Longitude;
},
(Array, idx) =>
{
return(new GeoCoordinateSimple()
{
Latitude = Array[idx],
Longitude = Array[idx + 1]
});
});
vertexCoordinateArray.CopyFrom(_coordinates, 0, 0, (vertexCount + 1));
vertexCoordinateArray.Dispose(); // written, get rid of it!
position = position + ((vertexCount + 1) * 2 * 4);
// write edges (each edge = 4 uints (16 bytes)).
var edgeArray = new MemoryMappedHugeArrayUInt32(file, edgeCount * EDGE_SIZE, edgeCount * EDGE_SIZE, 1024);
edgeArray.CopyFrom(_edges, edgeCount * EDGE_SIZE);
edgeArray.Dispose(); // written, get rid of it!
position = position + (edgeCount * EDGE_SIZE * 4);
// write edge data (each edgeData = edgeDataSize units (edgeDataSize * 4 bytes)).
var edgeDataArray = new MappedHugeArray <TEdgeData, uint>(
new MemoryMappedHugeArrayUInt32(file, edgeCount * edgeDataSize, edgeCount * edgeDataSize, 1024), edgeDataSize, mapTo, mapFrom);
edgeDataArray.CopyFrom(_edgeData, edgeCount);
edgeDataArray.Dispose(); // written, get rid of it!
position = position + (edgeCount * edgeDataSize * 4);
}
// write shapes.
stream.Seek(position, System.IO.SeekOrigin.Begin);
position = position + _edgeShapes.Serialize(new OsmSharp.IO.LimitedStream(stream));
return(position);
}
/// <summary>
/// Serializes this huge collection index to the given stream.
/// </summary>
/// <param name="stream"></param>
public long Serialize(Stream stream)
{
// first trim to minimize the amount of useless data store.
this.Trim();
// start writing.
long indexSize = _index.Length;
long coordinatesCount = _coordinates.Length;
long position = 0;
stream.Write(BitConverter.GetBytes(indexSize), 0, 8); // write the actual index size.
position = position + 8;
stream.Write(BitConverter.GetBytes(coordinatesCount), 0, 8); // write the actual number of coordinates.
position = position + 8;
// write in this order: index, shapes.
using (var file = new MemoryMappedStream(new LimitedStream(stream)))
{
// write index.
var indexArray = new MemoryMappedHugeArrayUInt64(file, indexSize, indexSize, 1024);
indexArray.CopyFrom(_index, indexSize);
indexArray.Dispose();
position = position + (indexSize * 8);
// write coordinates.
var coordinatesArray = new MemoryMappedHugeArraySingle(file, coordinatesCount * 2, coordinatesCount * 2, 1024);
coordinatesArray.CopyFrom(_coordinates);
coordinatesArray.Dispose();
position = position + (coordinatesCount * 2 * 4);
}
return position;
}
/// <summary>
/// Deserializes a huge collection index from the given stream.
/// </summary>
/// <param name="stream">The source stream.</param>
/// <param name="copy">The copy flag. When true all data is copied into memory, otherwise the source-stream is used as a memorymapped file.</param>
/// <returns></returns>
public static HugeCoordinateCollectionIndex Deserialize(Stream stream, bool copy = false)
{
// read sizes.
long position = 0;
var longBytes = new byte[8];
stream.Read(longBytes, 0, 8);
position = position + 8;
var indexLength = BitConverter.ToInt64(longBytes, 0);
stream.Read(longBytes, 0, 8);
position = position + 8;
var coordinateLength = BitConverter.ToInt64(longBytes, 0);
var file = new MemoryMappedStream(new LimitedStream(stream));
var indexArray = new MemoryMappedHugeArrayUInt64(file, indexLength, indexLength, 1024);
var coordinateArray = new MemoryMappedHugeArraySingle(file, coordinateLength * 2, coordinateLength * 2, 1024);
if (copy)
{ // copy the data.
var indexArrayCopy = new HugeArray<ulong>(indexLength);
indexArrayCopy.CopyFrom(indexArray);
var coordinateArrayCopy = new HugeArray<float>(coordinateLength * 2);
coordinateArrayCopy.CopyFrom(coordinateArray);
file.Dispose();
return new HugeCoordinateCollectionIndex(indexLength, indexArrayCopy, coordinateArrayCopy);
}
return new HugeCoordinateCollectionIndex(indexLength, indexArray, coordinateArray);
}
/// <summary>
/// Creates the stream processor associated with this command.
/// </summary>
/// <returns></returns>
public override ProcessorBase CreateProcessor()
{
// create output stream.
var graphStream = (new FileInfo(this.GraphFile)).Open(FileMode.Create);
// create memory mappped stream if option is there.
MemoryMappedStream memoryMappedStream = null;
if(!string.IsNullOrWhiteSpace(this.MemoryMapFile))
{
memoryMappedStream = new MemoryMappedStream((new FileInfo(this.MemoryMapFile)).Open(FileMode.Create));
}
switch(this.GraphType)
{
case GraphType.Regular:
return new ProcessorTarget(LiveEdgeFlatfileStreamTarget.CreateTarget(graphStream, memoryMappedStream));
case GraphType.Contracted:
return new ProcessorTarget(CHEdgeFlatfileStreamTarget.CreateTarget(graphStream, this.Vehicle, memoryMappedStream));
}
throw new InvalidCommandException("Invalid command: " + this.ToString());
}
