static void TestArray()
{
var count = 10_000_000L;
using (var mmf = new MemoryFile("list.db"))
{
var sw = Stopwatch.StartNew();
var arr = new MemoryArray <Int64>(mmf, count);
for (var i = 0; i < count; i++)
{
arr[i] = i;
}
sw.Stop();
var ms = sw.ElapsedMilliseconds;
Console.WriteLine("赋值[{0:n0}] {1:n0}tps", count, count * 1000L / ms);
sw.Restart();
for (var i = 0; i < count; i++)
{
var n = arr[i];
}
sw.Stop();
ms = sw.ElapsedMilliseconds;
Console.WriteLine("取值[{0:n0}] {1:n0}tps", count, count * 1000L / ms);
}
}
public void CompareToArrayTest()
{
var stringArrayRef = new string[1000];
var stringArray = new MemoryArray<string>(1000);
var randomGenerator = new System.Random(66707770); // make this deterministic
for (var idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Next(4) >= 2)
{ // add data.
stringArrayRef[idx] = idx.ToString();
stringArray[idx] = idx.ToString();
}
else
{
stringArrayRef[idx] = null;
stringArray[idx] = null;
}
}
for (var idx = 0; idx < 1000; idx++)
{
Assert.AreEqual(stringArrayRef[idx], stringArray[idx]);
}
}
/// <summary>
/// Deserializes a tags index from the given stream.
/// </summary>
public static AttributesIndex Deserialize(System.IO.Stream stream, bool copy = false,
AttributesIndexMode defaultIndexMode = AttributesIndexMode.ReverseStringIndexKeysOnly)
{
// read version byte.
long position = 1;
var version = stream.ReadByte();
int type = 0;
if (version < 2)
{ // unversioned version.
type = (byte)version;
}
else
{ // versioned.
// read the index mode.
var indexModeByte = stream.ReadByte();
position++;
defaultIndexMode = (AttributesIndexMode)indexModeByte;
// read the type.
type = stream.ReadByte();
position++;
}
// read the actual data.
long size;
if (type == 0)
{ // regular index.
var tagsIndex = Index <int[]> .CreateFromWithSize(stream, out size, !copy);
position += size + 8;
stream.Seek(position, System.IO.SeekOrigin.Begin);
var limitedStream = new LimitedStream(stream);
var stringIndex = Index <string> .CreateFromWithSize(limitedStream, out size, !copy);
position += size + 8;
stream.Seek(position, System.IO.SeekOrigin.Begin);
return(new AttributesIndex(defaultIndexMode, stringIndex, tagsIndex));
}
else
{ // increase one index.
var tagsIndex = Index <int[]> .CreateFromWithSize(stream, out size, !copy);
position += size + 8;
stream.Seek(position, System.IO.SeekOrigin.Begin);
var limitedStream = new LimitedStream(stream);
var stringIndex = Index <string> .CreateFromWithSize(limitedStream, out size, !copy);
position += size + 8;
stream.Seek(position, System.IO.SeekOrigin.Begin);
var indexLengthBytes = new byte[8];
stream.Read(indexLengthBytes, 0, 8);
var indexLength = BitConverter.ToInt64(indexLengthBytes, 0);
var index = new MemoryArray <uint>(indexLength);
index.CopyFrom(stream);
return(new AttributesIndex(defaultIndexMode, stringIndex, tagsIndex, index));
}
}
/// <summary>
/// Deserializes from the given stream, returns an optimized index.
/// </summary>
public static MappedAttributesIndex Deserialize(Stream stream, MappedAttributesIndexProfile profile)
{
var version = stream.ReadByte();
if (version > 1)
{
throw new Exception(
$"Cannot deserialize mapped attributes index: Invalid version #: {version}, upgrade Itinero.");
}
var bytes = new byte[4];
stream.Read(bytes, 0, 4);
var length = BitConverter.ToUInt32(bytes, 0);
ArrayBase <uint> data;
if (profile == null || profile.DataProfile == null)
{
data = new MemoryArray <uint>(length);
data.CopyFrom(stream);
}
else
{
var position = stream.Position;
var map = new MemoryMapStream(new CappedStream(stream, position,
length * 4));
data = new Array <uint>(map.CreateUInt32(length), profile.DataProfile);
stream.Seek(length * 4, SeekOrigin.Current);
}
var attributes = AttributesIndex.Deserialize(new LimitedStream(stream, stream.Position), true);
return(new MappedAttributesIndex(data, attributes));
}
/// <summary>
/// Deserializes this connection db from the given stream.
/// </summary>
public static ConnectionsDb Deserialize(Stream stream)
{
if (stream.ReadByte() != 1)
{
throw new Exception("Cannot deserialize db, version # doesn't match.");
}
var sortingByte = stream.ReadByte();
DefaultSorting?sorting = null;
if (sortingByte == 1)
{
sorting = DefaultSorting.DepartureTime;
}
else
{
sorting = DefaultSorting.ArrivalTime;
}
var binaryReader = new BinaryReader(stream);
var size = binaryReader.ReadInt64();
var connections = new MemoryArray <uint>(size * CONNECTION_SIZE);
connections.CopyFrom(stream);
var connectionsOrder = new MemoryArray <uint>(size);
connectionsOrder.CopyFrom(stream);
return(new ConnectionsDb(sorting, connections, connectionsOrder));
}
public void Compress(bool toReadonly, out long maxEdgeId)
{
this.Trim();
MemoryArray <uint> sortedVertices = new MemoryArray <uint>(this._vertices.Length / 2L);
for (uint index = 0; (long)index < ((ArrayBase <uint>)sortedVertices).Length; ++index)
{
((ArrayBase <uint>)sortedVertices)[(long)index] = index;
}
QuickSort.Sort((Func <long, long>)(i => (long)this._vertices[(long)(((ArrayBase <uint>)sortedVertices)[i] * 2U)] * ((ArrayBase <uint>)sortedVertices).Length + (long)((ArrayBase <uint>)sortedVertices)[i]), (Action <long, long>)((i, j) =>
{
uint num = ((ArrayBase <uint>)sortedVertices)[i];
((ArrayBase <uint>)sortedVertices)[i] = ((ArrayBase <uint>)sortedVertices)[j];
((ArrayBase <uint>)sortedVertices)[j] = num;
}), 0L, (long)(this.VertexCount - 1U));
uint num1 = 0;
for (uint index1 = 0; (long)index1 < ((ArrayBase <uint>)sortedVertices).Length; ++index1)
{
uint num2 = ((ArrayBase <uint>)sortedVertices)[(long)index1] * 2U;
uint num3 = this._vertices[(long)(num2 + 1U)];
uint num4 = this._vertices[(long)(num2 + 0U)] * (uint)this._edgeSize;
this._vertices[(long)(num2 + 0U)] = num1 / (uint)this._edgeSize;
uint num5 = 0;
while (num5 < num3 * (uint)this._edgeSize)
{
if ((int)num1 != (int)num4)
{
this._edges[(long)(num1 + num5)] = this._edges[(long)(num4 + num5)];
for (int index2 = 0; index2 < this._edgeDataSize; ++index2)
{
this._edges[(long)(uint)((int)num1 + (int)num5 + 1) + (long)index2] = this._edges[(long)(uint)((int)num4 + (int)num5 + 1) + (long)index2];
}
if (this._switchEdge != null)
{
this._switchEdge((uint)((ulong)(num4 + num5) / (ulong)this._edgeSize), (uint)((ulong)(num1 + num5) / (ulong)this._edgeSize));
}
}
num5 += (uint)this._edgeSize;
}
if (!toReadonly && num3 > 2U && ((int)num3 & (int)num3 - 1) != 0)
{
uint num6 = num3 | num3 >> 1;
uint num7 = num6 | num6 >> 2;
uint num8 = num7 | num7 >> 4;
uint num9 = num8 | num8 >> 8;
num3 = (num9 | num9 >> 16) + 1U;
}
num1 += num3 * (uint)this._edgeSize;
}
this._nextEdgePointer = num1;
this._readonly = toReadonly;
this._edges.Resize((long)this._nextEdgePointer);
maxEdgeId = this._edges.Length / (long)(uint)this._edgeSize;
}
/// <summary>
/// Deserializes from a stream.
/// </summary>
public static RoutingNetwork Deserialize(Stream stream, RoutingNetworkProfile profile)
{
var version = stream.ReadByte();
if (version > 2)
{
throw new Exception(string.Format("Cannot deserialize routing network: Invalid version #: {0}, upgrade Itinero.", version));
}
var position = stream.Position;
var initialPosition = stream.Position;
// read maxEdgeDistance if version # = 2.
var maxEdgeDistance = Constants.DefaultMaxEdgeDistance;
if (version == 2)
{
var bytes = new byte[4];
stream.Read(bytes, 0, 4);
maxEdgeDistance = BitConverter.ToSingle(bytes, 0);
}
// deserialize graph.
var graph = GeometricGraph.Deserialize(stream, profile == null ? null : profile.GeometricGraphProfile);
var size = stream.Position - position;
var edgeLength = graph.EdgeCount;
var edgeSize = 1;
ArrayBase <uint> edgeData;
if (profile == null)
{ // just create arrays and read the data.
edgeData = new MemoryArray <uint>(edgeLength * edgeSize);
edgeData.CopyFrom(stream);
size += edgeLength * edgeSize * 4;
}
else
{ // create accessors over the exact part of the stream that represents vertices/edges.
position = stream.Position;
var map = new MemoryMapStream(new CappedStream(stream, position,
edgeLength * edgeSize * 4));
edgeData = new Array <uint>(map.CreateUInt32(edgeLength * edgeSize), profile.EdgeDataProfile);
size += edgeLength * edgeSize * 4;
}
// make stream is positioned correctly.
stream.Seek(initialPosition + size, System.IO.SeekOrigin.Begin);
return(new RoutingNetwork(graph, edgeData, maxEdgeDistance));
}
/// <summary>
/// Deserializes from a stream.
/// </summary>
/// <returns></returns>
public static DirectedMetaGraph Deserialize(System.IO.Stream stream, DirectedMetaGraphProfile profile)
{
var version = stream.ReadByte();
if (version != 1)
{
throw new Exception(string.Format("Cannot deserialize directed meta graph: Invalid version #: {0}.", version));
}
var graph = DirectedGraph.Deserialize(stream, profile == null ? null : profile.DirectedGraphProfile);
var initialPosition = stream.Position;
long size = 0;
var bytes = new byte[4];
stream.Read(bytes, 0, 4);
size = size + 4;
var vertexSize = BitConverter.ToInt32(bytes, 0);
stream.Read(bytes, 0, 4);
size = size + 4;
var edgeSize = BitConverter.ToInt32(bytes, 0);
var edgeLength = graph.EdgeCount;
ArrayBase <uint> edges;
if (profile == null)
{ // just create arrays and read the data.
edges = new MemoryArray <uint>(edgeLength * edgeSize);
edges.CopyFrom(stream);
size += edgeLength * edgeSize * 4;
}
else
{ // create accessors over the exact part of the stream that represents vertices/edges.
var position = stream.Position;
var map1 = new MemoryMapStream(new CappedStream(stream, position,
edgeLength * edgeSize * 4));
edges = new Array <uint>(map1.CreateUInt32(edgeLength * edgeSize), profile.EdgeMetaProfile);
size += edgeLength * edgeSize * 4;
}
// make sure stream is positioned at the correct location.
stream.Seek(initialPosition + size, System.IO.SeekOrigin.Begin);
return(new DirectedMetaGraph(graph, edgeSize, edges));
}
/// <summary>
/// Deserializes an shapes index from the given stream.
/// </summary>
public static ShapesArray CreateFrom(Stream stream, bool copy, out long size)
{
var initialPosition = stream.Position;
size = 0;
var longBytes = new byte[8];
stream.Read(longBytes, 0, 8);
size += 8;
var indexLength = BitConverter.ToInt64(longBytes, 0);
stream.Read(longBytes, 0, 8);
size += 8;
var coordinatesLength = BitConverter.ToInt64(longBytes, 0);
ArrayBase <ulong> index;
ArrayBase <float> coordinates;
if (copy)
{ // just create arrays and read the data.
index = new MemoryArray <ulong>(indexLength);
index.CopyFrom(stream);
size += indexLength * 8;
coordinates = new MemoryArray <float>(coordinatesLength);
size += coordinatesLength * 4;
coordinates.CopyFrom(stream);
}
else
{ // create accessors over the exact part of the stream that represents vertices/edges.
var position = stream.Position;
var map1 = new MemoryMapStream(new CappedStream(stream, position, indexLength * 8));
index = new Array <ulong>(map1.CreateUInt64(indexLength));
size += indexLength * 8;
var map2 = new MemoryMapStream(new CappedStream(stream, position + indexLength * 8,
coordinatesLength * 4));
coordinates = new Array <float>(map2.CreateSingle(coordinatesLength));
size += coordinatesLength * 4;
}
// make stream is positioned correctly.
stream.Seek(initialPosition + size, System.IO.SeekOrigin.Begin);
return(new ShapesArray(index, coordinates));
}
/// <summary>
/// Deserializes a graph from the stream.
/// </summary>
public static GeometricGraph Deserialize(System.IO.Stream stream, GeometricGraphProfile profile)
{
var version = stream.ReadByte();
if (version != 1)
{
throw new Exception(string.Format("Cannot deserialize geometric graph: Invalid version #: {0}.", version));
}
var graph = Graph.Deserialize(stream, profile == null ? null : profile.GraphProfile);
var initialPosition = stream.Position;
var size = 0L;
ArrayBase <float> coordinates;
ShapesArray shapes;
if (profile == null)
{ // don't use the stream, the read from it.
coordinates = new MemoryArray <float>(graph.VertexCount * 2);
coordinates.CopyFrom(stream);
size += graph.VertexCount * 2 * 4;
long shapeSize;
shapes = ShapesArray.CreateFrom(stream, true, out shapeSize);
size += shapeSize;
}
else
{ // use the stream as a map.
var position = stream.Position;
var map = new MemoryMapStream(new CappedStream(stream, position, graph.VertexCount * 4 * 2));
coordinates = new Array <float>(map.CreateSingle(graph.VertexCount * 2), profile.CoordinatesProfile);
size += graph.VertexCount * 2 * 4;
stream.Seek(position + graph.VertexCount * 4 * 2, System.IO.SeekOrigin.Begin);
long shapeSize;
shapes = ShapesArray.CreateFrom(stream, false, out shapeSize);
size += shapeSize;
}
// make stream is positioned correctly.
stream.Seek(initialPosition + size, System.IO.SeekOrigin.Begin);
return(new GeometricGraph(graph, coordinates, shapes));
}
public void TestWriteToAndReadFromVariable()
{
using (var memoryStream = new MemoryStream())
{
var array = new MemoryArray <string>(1000);
for (var i = 0; i < array.Length; i++)
{
array[i] = (i + 100).ToString();
}
array.CopyToWithSize(memoryStream);
memoryStream.Seek(0, SeekOrigin.Begin);
var array1 = MemoryArray <string> .CopyFromWithSize(memoryStream);
for (var i = 0; i < array.Length; i++)
{
Assert.AreEqual(array[i], array1[i]);
}
}
}
public void ResizeTest()
{
var stringArrayRef = new string[1000];
var stringArray = new MemoryArray<string>(1000);
var randomGenerator = new System.Random(66707770); // make this deterministic
for (int idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Next(4) >= 2)
{ // add data.
stringArrayRef[idx] = idx.ToString();
stringArray[idx] = idx.ToString();
}
else
{
stringArrayRef[idx] = null;
stringArray[idx] = null;
}
}
Array.Resize<string>(ref stringArrayRef, 335);
stringArray.Resize(335);
Assert.AreEqual(stringArrayRef.Length, stringArray.Length);
for (int idx = 0; idx < stringArrayRef.Length; idx++)
{
Assert.AreEqual(stringArrayRef[idx], stringArray[idx]);
}
stringArrayRef = new string[1000];
stringArray = new MemoryArray<string>(1000);
for (int idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Next(4) >= 2)
{ // add data.
stringArrayRef[idx] = idx.ToString();
stringArray[idx] = idx.ToString();
}
else
{
stringArrayRef[idx] = null;
stringArray[idx] = null;
}
}
Array.Resize<string>(ref stringArrayRef, 1235);
stringArray.Resize(1235);
Assert.AreEqual(stringArrayRef.Length, stringArray.Length);
for (int idx = 0; idx < stringArrayRef.Length; idx++)
{
Assert.AreEqual(stringArrayRef[idx], stringArray[idx]);
}
}