public void HugeArraySimpleTest()
{
var stringArrayRef = new string[1000];
var stringArray = new HugeArray <string>(1000, 300);
var randomGenerator = new RandomGenerator(66707770); // make this deterministic
for (int idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Generate(2.0) > 1)
{ // add data.
stringArrayRef[idx] = idx.ToString();
stringArray[idx] = idx.ToString();
}
else
{
stringArrayRef[idx] = null;
stringArray[idx] = null;
}
}
for (int idx = 0; idx < 1000; idx++)
{
Assert.AreEqual(stringArrayRef[idx], stringArray[idx]);
}
}
/// <summary>
/// Deserializes a huge collection index from the given stream.
/// </summary>
/// <param name="stream">The source stream.</param>
/// <param name="copy">The copy flag. When true all data is copied into memory, otherwise the source-stream is used as a memorymapped file.</param>
/// <returns></returns>
public static HugeCoordinateCollectionIndex Deserialize(Stream stream, bool copy = false)
{
// read sizes.
long position = 0;
var longBytes = new byte[8];
stream.Read(longBytes, 0, 8);
position = position + 8;
var indexLength = BitConverter.ToInt64(longBytes, 0);
stream.Read(longBytes, 0, 8);
position = position + 8;
var coordinateLength = BitConverter.ToInt64(longBytes, 0);
var file = new MemoryMappedStream(new LimitedStream(stream));
var indexArray = new MemoryMappedHugeArrayUInt64(file, indexLength, indexLength, 1024);
var coordinateArray = new MemoryMappedHugeArraySingle(file, coordinateLength * 2, coordinateLength * 2, 1024);
if (copy)
{ // copy the data.
var indexArrayCopy = new HugeArray <ulong>(indexLength);
indexArrayCopy.CopyFrom(indexArray);
var coordinateArrayCopy = new HugeArray <float>(coordinateLength * 2);
coordinateArrayCopy.CopyFrom(coordinateArray);
file.Dispose();
return(new HugeCoordinateCollectionIndex(indexLength, indexArrayCopy, coordinateArrayCopy));
}
return(new HugeCoordinateCollectionIndex(indexLength, indexArray, coordinateArray));
}
public void CanAssignToHugeArrayElements()
{
ulong arraySize = 2L * Int32.MaxValue;
HugeArray <int> array = new HugeArray <int>(arraySize);
ulong[] indices = new ulong[10000];
int scale = 0, value = 0;
ulong v = 0;
for (int i = 0; i < indices.Length; i++)
{
v = indices[0];
while (indices.Contains(v))
{
scale = rng.Next(1, 2);
value = rng.Next(0, Int32.MaxValue - 1);
v = (ulong)scale * (ulong)value;
}
array[v] = i;
indices[i] = v;
}
for (int i = 0; i < indices.Length; i++)
{
Assert.Equal(i, array[indices[i]]);
}
array.Close();
}
public void CanConvertToVector()
{
HugeArray <uint> a = new HugeArray <uint>(8, 1, 11, 94, 5, 0, 0, 0, 8);
Vector <uint> v = a.AcquireAsSingleVector();
Assert.Equal(a[0], v[0]);
Assert.Equal(a[3], v[3]);
Assert.Equal(a[7], v[7]);
HugeArray <uint> a2 = new HugeArray <uint>(12, 11, 112, 594, 65, 0, 0, 0, 8, 14, 90, 2, 8);
Vector <uint> v2 = a2.AcquireSliceToVector(0);
Assert.Equal(11u, v2[0]);
Assert.Equal(8u, v2[7]);
HugeArray <uint> a3 = new HugeArray <uint>((ulong)Int32.MaxValue + 10000);
a3.Fill(7u);
a3[(ulong)Int32.MaxValue + 100] = 9;
a3[(ulong)Int32.MaxValue + 101] = 4;
Vector <uint> v3 = a3.AcquireSliceToVector((ulong)Int32.MaxValue + 99);
Assert.Equal(9u, v3[1]);
Assert.Equal(4u, v3[2]);
Assert.Equal(a3[(ulong)Int32.MaxValue + 99], v3[0]);
Assert.Equal(7u, v3[0]);
Assert.Equal(7u, v3[7]);
a.Close();
a2.Close();
a3.Close();
}
/// <summary>
/// Sorts the vertices in the given graph based on a hilbert curve using the default step count.
/// </summary>
/// <typeparam name="TEdgeData"></typeparam>
public static HugeArrayBase <uint> BuildHilbertRank <TEdgeData>(this GraphBase <TEdgeData> graph, int n)
where TEdgeData : struct, IGraphEdgeData
{
var ranks = new HugeArray <uint>(graph.VertexCount + 1);
graph.BuildHilbertRank(n, ranks);
return(ranks);
}
public void CanFill()
{
HugeArray <int> array = new HugeArray <int>(1000);
array.Fill(33);
Assert.Equal(33, array[999]);
array.Close();
}
public void FillHugeNativeArray()
{
T fill = GetArrayFillValue();
HugeArray <T> array = new HugeArray <T>(ArraySize);
array.Fill(fill);
T r = array[ArraySize / 2];
}
/// <summary>
/// Sorts the vertices in the given graph based on a hilbert curve using the default step count.
/// </summary>
/// <typeparam name="TEdgeData"></typeparam>
public static HugeArrayBase <uint> BuildHilbertRank <TEdgeData>(this GraphBase <TEdgeData> graph)
where TEdgeData : struct, IGraphEdgeData
{
var ranks = new HugeArray <uint>(graph.VertexCount + 1);
graph.BuildHilbertRank(GraphExtensions.DefaultHilbertSteps, ranks);
return(ranks);
}
public void GlobalSetup()
{
managedArray = new T[2146435071];
nativeArray = new HugeArray <T>(ArraySize);
fill = GetArrayFillValue();
mul = GetArrayMulValue();
Console.WriteLine("Managed array fill value is {0}.", fill);
Console.WriteLine("Multiply factor is {0}", mul);
}
public void FillHugeNativeArray()
{
T fill = GM <T> .Random();
HugeArray <T> array = new HugeArray <T>(ArraySize);
array.Fill(fill);
T r = array[ArraySize / 2];
}
public void HugeArrayResizeTests()
{
var stringArrayRef = new string[1000];
var stringArray = new HugeArray <string>(1000, 300);
var randomGenerator = new RandomGenerator(66707770); // make this deterministic
for (int idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Generate(2.0) > 1)
{ // add data.
stringArrayRef[idx] = idx.ToString();
stringArray[idx] = idx.ToString();
}
else
{
stringArrayRef[idx] = null;
stringArray[idx] = null;
}
}
Array.Resize <string>(ref stringArrayRef, 335);
stringArray.Resize(335);
Assert.AreEqual(stringArrayRef.Length, stringArray.Length);
for (int idx = 0; idx < stringArrayRef.Length; idx++)
{
Assert.AreEqual(stringArrayRef[idx], stringArray[idx]);
}
stringArrayRef = new string[1000];
stringArray = new HugeArray <string>(1000, 300);
for (int idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Generate(2.0) > 1)
{ // add data.
stringArrayRef[idx] = idx.ToString();
stringArray[idx] = idx.ToString();
}
else
{
stringArrayRef[idx] = null;
stringArray[idx] = null;
}
}
Array.Resize <string>(ref stringArrayRef, 1235);
stringArray.Resize(1235);
Assert.AreEqual(stringArrayRef.Length, stringArray.Length);
for (int idx = 0; idx < stringArrayRef.Length; idx++)
{
Assert.AreEqual(stringArrayRef[idx], stringArray[idx]);
}
}
public override void GlobalSetup()
{
base.GlobalSetup();
managedArray = new T[2146435071];
nativeArray = new HugeArray <T>(ArraySize);
(mul, fill) = GM <T> .RandomMultiplyFactorAndValue();
Console.WriteLine("Managed array fill value is {0}.", fill);
Console.WriteLine("Multiply factor is {0}", mul);
}
public void CanConstructHugeArray()
{
ulong arraySize = 2L * Int32.MaxValue;
ulong point = 1L * Int32.MaxValue;
HugeArray <int> array = new HugeArray <int>(arraySize);
array[point] = 1;
Assert.Equal(1, array[point]);
array.Close();
}
public void HugeArrayResizeTests()
{
var stringArrayRef = new string[1000];
var stringArray = new HugeArray<string>(1000, 300);
var randomGenerator = new RandomGenerator(66707770); // make this deterministic
for (int idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Generate(2.0) > 1)
{ // add data.
stringArrayRef[idx] = idx.ToString();
stringArray[idx] = idx.ToString();
}
else
{
stringArrayRef[idx] = null;
stringArray[idx] = null;
}
}
Array.Resize<string>(ref stringArrayRef, 335);
stringArray.Resize(335);
Assert.AreEqual(stringArrayRef.Length, stringArray.Length);
for (int idx = 0; idx < stringArrayRef.Length; idx++)
{
Assert.AreEqual(stringArrayRef[idx], stringArray[idx]);
}
stringArrayRef = new string[1000];
stringArray = new HugeArray<string>(1000, 300);
for (int idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Generate(2.0) > 1)
{ // add data.
stringArrayRef[idx] = idx.ToString();
stringArray[idx] = idx.ToString();
}
else
{
stringArrayRef[idx] = null;
stringArray[idx] = null;
}
}
Array.Resize<string>(ref stringArrayRef, 1235);
stringArray.Resize(1235);
Assert.AreEqual(stringArrayRef.Length, stringArray.Length);
for (int idx = 0; idx < stringArrayRef.Length; idx++)
{
Assert.AreEqual(stringArrayRef[idx], stringArray[idx]);
}
}
public void Test1()
{
{
var a = new HugeArray <string>(0);
Assert.AreEqual(a.Count, 0);
Assert.AreEqual(a.BlockSize, 256);
Assert.AreEqual(a.BlockBits, 8);
Assert.AreEqual(a.BlockMask, 255);
Assert.IsTrue(a[0] == null);
a[1] = "1";
Assert.AreEqual(a.Count, 2);
Assert.AreEqual(a.BlockCount, 1);
Assert.IsTrue(a[0] == null);
Assert.IsTrue(a[1].Equals("1"));
Assert.IsTrue(a.GetOrAdd(0, () => "0").Equals("0"));
Assert.IsTrue(a[0].Equals("0"));
a[257] = "257";
Assert.AreEqual(a.Count, 258);
Assert.AreEqual(a.BlockCount, 2);
Assert.IsTrue(a[255] == null);
Assert.IsTrue(a[256] == null);
Assert.IsTrue(a[257].Equals("257"));
a[-1] = "-1";
Assert.AreEqual(a.Count, 258 + 1);
Assert.AreEqual(a.BlockCount, 3);
Assert.IsTrue(a[-1].Equals("-1"));
a[-256] = "-256";
Assert.AreEqual(a.Count, 258 + 256);
Assert.AreEqual(a.BlockCount, 3);
Assert.IsTrue(a[-256].Equals("-256"));
a[-257] = "-257";
Assert.AreEqual(a.Count, 258 + 257);
Assert.AreEqual(a.BlockCount, 4);
Assert.IsTrue(a[-257].Equals("-257"));
}
{
var a = new HugeArray <string>(256);
Assert.AreEqual(a.Count, 0);
Assert.AreEqual(a.BlockSize, 256);
Assert.AreEqual(a.BlockBits, 8);
Assert.AreEqual(a.BlockMask, 255);
}
{
var a = new HugeArray <string>(900);
Assert.AreEqual(a.Count, 0);
Assert.AreEqual(a.BlockSize, 1024);
Assert.AreEqual(a.BlockBits, 10);
Assert.AreEqual(a.BlockMask, 1023);
}
}
public void CanVectorizedMultiply()
{
HugeArray <uint> a = new HugeArray <uint>(8, 1, 2, 3, 4, 5, 6, 7, 8);
HugeArray <int> b = new HugeArray <int>(8, 111, 22, 345, 40888, 3, 777, 99, 6);
a.VectorMultiply(2);
Assert.Equal(2u, a[0]);
Assert.Equal(8u, a[3]);
Assert.Equal(16u, a[7]);
b.VectorMultiply(6);
Assert.Equal(666, b[0]);
}
/// <summary>
/// Initializes the reverse index.
/// </summary>
private void BuildReverse()
{
_reverseDirectNeighbours = new HugeArray <uint>(_graph.VertexCount + 1);
_additionalReverseNeighbours = new Dictionary <uint, uint[]>();
for (uint currentVertex = 1; currentVertex <= _graph.VertexCount; currentVertex++)
{
var edges = _graph.GetEdges(currentVertex);
while (edges.MoveNext())
{
uint neighbour = edges.Neighbour;
this.AddReverse(neighbour, currentVertex);
}
}
}
/// <summary>
/// Copies all data from the given graph.
/// </summary>
/// <typeparam name="TEdgeData"></typeparam>
public static void SortHilbert <TEdgeData>(this GraphBase <TEdgeData> graph, int n, Action <uint, uint> transform)
where TEdgeData : struct, IGraphEdgeData
{
// build ranks.
var ranks = graph.BuildHilbertRank(n);
// invert ranks.
var transformations = new HugeArray <uint>(ranks.Length);
for (uint i = 0; i < ranks.Length; i++)
{
if (transform != null)
{
transform(ranks[i], i);
}
transformations[ranks[i]] = i;
}
// copy from the given graph but with sorted vertices.
graph.Sort(transformations);
}
/// <summary>
/// Moves data around to obtain the smallest possible datastructure still containing all data.
/// </summary>
public void Compress()
{
// trim first.
this.Trim();
// build a list of all vertices sorted by their first position.
var sortedVertices = new HugeArray<uint>(_index.Length);
for (uint i = 0; i < sortedVertices.Length; i++)
{
sortedVertices[i] = i;
}
// sort vertices and coordinates.
QuickSort.Sort((i) => ((long)(_index[i] / (ulong)MAX_COLLECTION_SIZE)) * 2, (i, j) =>
{
var tempRef = sortedVertices[i];
sortedVertices[i] = sortedVertices[j];
sortedVertices[j] = tempRef;
}, 0, _index.Length - 1);
// move data down.
uint pointer = 0;
for (uint i = 0; i < sortedVertices.Length; i++)
{
long index, size;
if(this.TryGetIndexAndSize(sortedVertices[i], out index, out size))
{
if(size > 0)
{
var newIndex = pointer / 2;
for(var c = 0; c < size; c++)
{
_coordinates[pointer] = _coordinates[(index + c) * 2];
pointer++;
_coordinates[pointer] = _coordinates[(index + c) * 2 + 1];
pointer++;
}
_index[sortedVertices[i]] = (ulong)(newIndex * MAX_COLLECTION_SIZE) + (ulong)size;
}
}
}
_nextIdx = pointer / 2;
// trim again, something may have changed.
this.Trim();
}
/// <summary>
/// Deserializes a graph from the given stream.
/// </summary>
/// <param name="stream">The stream to read from. Reading will start at position 0.</param>
/// <param name="edgeDataSize">The edge data size.</param>
/// <param name="mapFrom">The map from for the edge data.</param>
/// <param name="mapTo">The map to for the edge data.</param>
/// <param name="copy">Flag to make an in-memory copy.</param>
/// <returns></returns>
public new static DirectedGraph <TEdgeData> Deserialize(System.IO.Stream stream, int edgeDataSize, MappedHugeArray <TEdgeData, uint> .MapFrom mapFrom,
MappedHugeArray <TEdgeData, uint> .MapTo mapTo, bool copy)
{
// read sizes.
long position = 0;
stream.Seek(4, System.IO.SeekOrigin.Begin);
position = position + 4;
var longBytes = new byte[8];
stream.Read(longBytes, 0, 8);
position = position + 8;
var vertexLength = BitConverter.ToInt64(longBytes, 0);
stream.Read(longBytes, 0, 8);
position = position + 8;
var edgeLength = BitConverter.ToInt64(longBytes, 0);
var bufferSize = 32;
var cacheSize = MemoryMappedHugeArrayUInt32.DefaultCacheSize;
var file = new MemoryMappedStream(new OsmSharp.IO.LimitedStream(stream));
var vertexArray = new MemoryMappedHugeArrayUInt32(file, (vertexLength + 1) * VERTEX_SIZE, (vertexLength + 1) * VERTEX_SIZE, bufferSize / 4, cacheSize * 4);
position = position + ((vertexLength + 1) * VERTEX_SIZE * 4);
var vertexCoordinateArray = new MappedHugeArray <GeoCoordinateSimple, float>(
new MemoryMappedHugeArraySingle(file, (vertexLength + 1) * 2, (vertexLength + 1) * 2, bufferSize / 4, cacheSize * 4),
2, (array, idx, coordinate) =>
{
array[idx] = coordinate.Latitude;
array[idx + 1] = coordinate.Longitude;
},
(Array, idx) =>
{
return(new GeoCoordinateSimple()
{
Latitude = Array[idx],
Longitude = Array[idx + 1]
});
});
position = position + ((vertexLength + 1) * 2 * 4);
var edgeArray = new MemoryMappedHugeArrayUInt32(file, edgeLength * EDGE_SIZE, edgeLength * EDGE_SIZE, bufferSize / 2, cacheSize * 4);
position = position + (edgeLength * EDGE_SIZE * 4);
var edgeDataArray = new MappedHugeArray <TEdgeData, uint>(
new MemoryMappedHugeArrayUInt32(file, edgeLength * edgeDataSize, edgeLength * edgeDataSize, bufferSize * 2, cacheSize * 2), edgeDataSize, mapTo, mapFrom);
position = position + (edgeLength * edgeDataSize * 4);
// deserialize shapes.
stream.Seek(position, System.IO.SeekOrigin.Begin);
var cappedStream = new OsmSharp.IO.LimitedStream(stream);
var shapes = HugeCoordinateCollectionIndex.Deserialize(cappedStream, copy);
if (copy)
{ // copy the data.
var vertexArrayCopy = new HugeArray <uint>(vertexArray.Length);
vertexArrayCopy.CopyFrom(vertexArray);
var vertexCoordinateArrayCopy = new HugeArray <GeoCoordinateSimple>(vertexCoordinateArray.Length);
vertexCoordinateArrayCopy.CopyFrom(vertexCoordinateArray);
var edgeArrayCopy = new HugeArray <uint>(edgeArray.Length);
edgeArrayCopy.CopyFrom(edgeArray);
var edgeDataArrayCopy = new HugeArray <TEdgeData>(edgeDataArray.Length);
edgeDataArrayCopy.CopyFrom(edgeDataArray);
file.Dispose();
return(new DirectedGraph <TEdgeData>(vertexCoordinateArrayCopy, vertexArrayCopy, edgeArrayCopy, edgeDataArrayCopy, shapes));
}
return(new DirectedGraph <TEdgeData>(vertexCoordinateArray, vertexArray, edgeArray, edgeDataArray, shapes));
}
public void HugeArraySimpleTest()
{
var stringArrayRef = new string[1000];
var stringArray = new HugeArray<string>(1000, 300);
var randomGenerator = new RandomGenerator(66707770); // make this deterministic
for (int idx = 0; idx < 1000; idx++)
{
if (randomGenerator.Generate(2.0) > 1)
{ // add data.
stringArrayRef[idx] = idx.ToString();
stringArray[idx] = idx.ToString();
}
else
{
stringArrayRef[idx] = null;
stringArray[idx] = null;
}
}
for (int idx = 0; idx < 1000; idx++)
{
Assert.AreEqual(stringArrayRef[idx], stringArray[idx]);
}
}
public void Init()
{
hg = new HugeArray <long>(40);
}
/// <summary>
/// Deserializes a huge collection index from the given stream.
/// </summary>
/// <param name="stream">The source stream.</param>
/// <param name="copy">The copy flag. When true all data is copied into memory, otherwise the source-stream is used as a memorymapped file.</param>
/// <returns></returns>
public static HugeCoordinateCollectionIndex Deserialize(Stream stream, bool copy = false)
{
// read sizes.
long position = 0;
var longBytes = new byte[8];
stream.Read(longBytes, 0, 8);
position = position + 8;
var indexLength = BitConverter.ToInt64(longBytes, 0);
stream.Read(longBytes, 0, 8);
position = position + 8;
var coordinateLength = BitConverter.ToInt64(longBytes, 0);
var file = new MemoryMappedStream(new LimitedStream(stream));
var indexArray = new MemoryMappedHugeArrayUInt64(file, indexLength, indexLength, 1024);
var coordinateArray = new MemoryMappedHugeArraySingle(file, coordinateLength * 2, coordinateLength * 2, 1024);
if (copy)
{ // copy the data.
var indexArrayCopy = new HugeArray<ulong>(indexLength);
indexArrayCopy.CopyFrom(indexArray);
var coordinateArrayCopy = new HugeArray<float>(coordinateLength * 2);
coordinateArrayCopy.CopyFrom(coordinateArray);
file.Dispose();
return new HugeCoordinateCollectionIndex(indexLength, indexArrayCopy, coordinateArrayCopy);
}
return new HugeCoordinateCollectionIndex(indexLength, indexArray, coordinateArray);
}
public void CreateHugeNativeArray()
{
HugeArray <T> array = new HugeArray <T>(ArraySize);
array.Close();
}
