public void HugeArraySimpleTest() { var stringArrayRef = new string[1000]; var stringArray = new HugeArray <string>(1000, 300); var randomGenerator = new RandomGenerator(66707770); // make this deterministic for (int idx = 0; idx < 1000; idx++) { if (randomGenerator.Generate(2.0) > 1) { // add data. stringArrayRef[idx] = idx.ToString(); stringArray[idx] = idx.ToString(); } else { stringArrayRef[idx] = null; stringArray[idx] = null; } } for (int idx = 0; idx < 1000; idx++) { Assert.AreEqual(stringArrayRef[idx], stringArray[idx]); } }
/// <summary> /// Deserializes a huge collection index from the given stream. /// </summary> /// <param name="stream">The source stream.</param> /// <param name="copy">The copy flag. When true all data is copied into memory, otherwise the source-stream is used as a memorymapped file.</param> /// <returns></returns> public static HugeCoordinateCollectionIndex Deserialize(Stream stream, bool copy = false) { // read sizes. long position = 0; var longBytes = new byte[8]; stream.Read(longBytes, 0, 8); position = position + 8; var indexLength = BitConverter.ToInt64(longBytes, 0); stream.Read(longBytes, 0, 8); position = position + 8; var coordinateLength = BitConverter.ToInt64(longBytes, 0); var file = new MemoryMappedStream(new LimitedStream(stream)); var indexArray = new MemoryMappedHugeArrayUInt64(file, indexLength, indexLength, 1024); var coordinateArray = new MemoryMappedHugeArraySingle(file, coordinateLength * 2, coordinateLength * 2, 1024); if (copy) { // copy the data. var indexArrayCopy = new HugeArray <ulong>(indexLength); indexArrayCopy.CopyFrom(indexArray); var coordinateArrayCopy = new HugeArray <float>(coordinateLength * 2); coordinateArrayCopy.CopyFrom(coordinateArray); file.Dispose(); return(new HugeCoordinateCollectionIndex(indexLength, indexArrayCopy, coordinateArrayCopy)); } return(new HugeCoordinateCollectionIndex(indexLength, indexArray, coordinateArray)); }
public void CanAssignToHugeArrayElements() { ulong arraySize = 2L * Int32.MaxValue; HugeArray <int> array = new HugeArray <int>(arraySize); ulong[] indices = new ulong[10000]; int scale = 0, value = 0; ulong v = 0; for (int i = 0; i < indices.Length; i++) { v = indices[0]; while (indices.Contains(v)) { scale = rng.Next(1, 2); value = rng.Next(0, Int32.MaxValue - 1); v = (ulong)scale * (ulong)value; } array[v] = i; indices[i] = v; } for (int i = 0; i < indices.Length; i++) { Assert.Equal(i, array[indices[i]]); } array.Close(); }
public void CanConvertToVector() { HugeArray <uint> a = new HugeArray <uint>(8, 1, 11, 94, 5, 0, 0, 0, 8); Vector <uint> v = a.AcquireAsSingleVector(); Assert.Equal(a[0], v[0]); Assert.Equal(a[3], v[3]); Assert.Equal(a[7], v[7]); HugeArray <uint> a2 = new HugeArray <uint>(12, 11, 112, 594, 65, 0, 0, 0, 8, 14, 90, 2, 8); Vector <uint> v2 = a2.AcquireSliceToVector(0); Assert.Equal(11u, v2[0]); Assert.Equal(8u, v2[7]); HugeArray <uint> a3 = new HugeArray <uint>((ulong)Int32.MaxValue + 10000); a3.Fill(7u); a3[(ulong)Int32.MaxValue + 100] = 9; a3[(ulong)Int32.MaxValue + 101] = 4; Vector <uint> v3 = a3.AcquireSliceToVector((ulong)Int32.MaxValue + 99); Assert.Equal(9u, v3[1]); Assert.Equal(4u, v3[2]); Assert.Equal(a3[(ulong)Int32.MaxValue + 99], v3[0]); Assert.Equal(7u, v3[0]); Assert.Equal(7u, v3[7]); a.Close(); a2.Close(); a3.Close(); }
/// <summary> /// Sorts the vertices in the given graph based on a hilbert curve using the default step count. /// </summary> /// <typeparam name="TEdgeData"></typeparam> public static HugeArrayBase <uint> BuildHilbertRank <TEdgeData>(this GraphBase <TEdgeData> graph, int n) where TEdgeData : struct, IGraphEdgeData { var ranks = new HugeArray <uint>(graph.VertexCount + 1); graph.BuildHilbertRank(n, ranks); return(ranks); }
public void CanFill() { HugeArray <int> array = new HugeArray <int>(1000); array.Fill(33); Assert.Equal(33, array[999]); array.Close(); }
public void FillHugeNativeArray() { T fill = GetArrayFillValue(); HugeArray <T> array = new HugeArray <T>(ArraySize); array.Fill(fill); T r = array[ArraySize / 2]; }
/// <summary> /// Sorts the vertices in the given graph based on a hilbert curve using the default step count. /// </summary> /// <typeparam name="TEdgeData"></typeparam> public static HugeArrayBase <uint> BuildHilbertRank <TEdgeData>(this GraphBase <TEdgeData> graph) where TEdgeData : struct, IGraphEdgeData { var ranks = new HugeArray <uint>(graph.VertexCount + 1); graph.BuildHilbertRank(GraphExtensions.DefaultHilbertSteps, ranks); return(ranks); }
public void GlobalSetup() { managedArray = new T[2146435071]; nativeArray = new HugeArray <T>(ArraySize); fill = GetArrayFillValue(); mul = GetArrayMulValue(); Console.WriteLine("Managed array fill value is {0}.", fill); Console.WriteLine("Multiply factor is {0}", mul); }
public void FillHugeNativeArray() { T fill = GM <T> .Random(); HugeArray <T> array = new HugeArray <T>(ArraySize); array.Fill(fill); T r = array[ArraySize / 2]; }
public void HugeArrayResizeTests() { var stringArrayRef = new string[1000]; var stringArray = new HugeArray <string>(1000, 300); var randomGenerator = new RandomGenerator(66707770); // make this deterministic for (int idx = 0; idx < 1000; idx++) { if (randomGenerator.Generate(2.0) > 1) { // add data. stringArrayRef[idx] = idx.ToString(); stringArray[idx] = idx.ToString(); } else { stringArrayRef[idx] = null; stringArray[idx] = null; } } Array.Resize <string>(ref stringArrayRef, 335); stringArray.Resize(335); Assert.AreEqual(stringArrayRef.Length, stringArray.Length); for (int idx = 0; idx < stringArrayRef.Length; idx++) { Assert.AreEqual(stringArrayRef[idx], stringArray[idx]); } stringArrayRef = new string[1000]; stringArray = new HugeArray <string>(1000, 300); for (int idx = 0; idx < 1000; idx++) { if (randomGenerator.Generate(2.0) > 1) { // add data. stringArrayRef[idx] = idx.ToString(); stringArray[idx] = idx.ToString(); } else { stringArrayRef[idx] = null; stringArray[idx] = null; } } Array.Resize <string>(ref stringArrayRef, 1235); stringArray.Resize(1235); Assert.AreEqual(stringArrayRef.Length, stringArray.Length); for (int idx = 0; idx < stringArrayRef.Length; idx++) { Assert.AreEqual(stringArrayRef[idx], stringArray[idx]); } }
public override void GlobalSetup() { base.GlobalSetup(); managedArray = new T[2146435071]; nativeArray = new HugeArray <T>(ArraySize); (mul, fill) = GM <T> .RandomMultiplyFactorAndValue(); Console.WriteLine("Managed array fill value is {0}.", fill); Console.WriteLine("Multiply factor is {0}", mul); }
public void CanConstructHugeArray() { ulong arraySize = 2L * Int32.MaxValue; ulong point = 1L * Int32.MaxValue; HugeArray <int> array = new HugeArray <int>(arraySize); array[point] = 1; Assert.Equal(1, array[point]); array.Close(); }
public void HugeArrayResizeTests() { var stringArrayRef = new string[1000]; var stringArray = new HugeArray<string>(1000, 300); var randomGenerator = new RandomGenerator(66707770); // make this deterministic for (int idx = 0; idx < 1000; idx++) { if (randomGenerator.Generate(2.0) > 1) { // add data. stringArrayRef[idx] = idx.ToString(); stringArray[idx] = idx.ToString(); } else { stringArrayRef[idx] = null; stringArray[idx] = null; } } Array.Resize<string>(ref stringArrayRef, 335); stringArray.Resize(335); Assert.AreEqual(stringArrayRef.Length, stringArray.Length); for (int idx = 0; idx < stringArrayRef.Length; idx++) { Assert.AreEqual(stringArrayRef[idx], stringArray[idx]); } stringArrayRef = new string[1000]; stringArray = new HugeArray<string>(1000, 300); for (int idx = 0; idx < 1000; idx++) { if (randomGenerator.Generate(2.0) > 1) { // add data. stringArrayRef[idx] = idx.ToString(); stringArray[idx] = idx.ToString(); } else { stringArrayRef[idx] = null; stringArray[idx] = null; } } Array.Resize<string>(ref stringArrayRef, 1235); stringArray.Resize(1235); Assert.AreEqual(stringArrayRef.Length, stringArray.Length); for (int idx = 0; idx < stringArrayRef.Length; idx++) { Assert.AreEqual(stringArrayRef[idx], stringArray[idx]); } }
public void Test1() { { var a = new HugeArray <string>(0); Assert.AreEqual(a.Count, 0); Assert.AreEqual(a.BlockSize, 256); Assert.AreEqual(a.BlockBits, 8); Assert.AreEqual(a.BlockMask, 255); Assert.IsTrue(a[0] == null); a[1] = "1"; Assert.AreEqual(a.Count, 2); Assert.AreEqual(a.BlockCount, 1); Assert.IsTrue(a[0] == null); Assert.IsTrue(a[1].Equals("1")); Assert.IsTrue(a.GetOrAdd(0, () => "0").Equals("0")); Assert.IsTrue(a[0].Equals("0")); a[257] = "257"; Assert.AreEqual(a.Count, 258); Assert.AreEqual(a.BlockCount, 2); Assert.IsTrue(a[255] == null); Assert.IsTrue(a[256] == null); Assert.IsTrue(a[257].Equals("257")); a[-1] = "-1"; Assert.AreEqual(a.Count, 258 + 1); Assert.AreEqual(a.BlockCount, 3); Assert.IsTrue(a[-1].Equals("-1")); a[-256] = "-256"; Assert.AreEqual(a.Count, 258 + 256); Assert.AreEqual(a.BlockCount, 3); Assert.IsTrue(a[-256].Equals("-256")); a[-257] = "-257"; Assert.AreEqual(a.Count, 258 + 257); Assert.AreEqual(a.BlockCount, 4); Assert.IsTrue(a[-257].Equals("-257")); } { var a = new HugeArray <string>(256); Assert.AreEqual(a.Count, 0); Assert.AreEqual(a.BlockSize, 256); Assert.AreEqual(a.BlockBits, 8); Assert.AreEqual(a.BlockMask, 255); } { var a = new HugeArray <string>(900); Assert.AreEqual(a.Count, 0); Assert.AreEqual(a.BlockSize, 1024); Assert.AreEqual(a.BlockBits, 10); Assert.AreEqual(a.BlockMask, 1023); } }
public void CanVectorizedMultiply() { HugeArray <uint> a = new HugeArray <uint>(8, 1, 2, 3, 4, 5, 6, 7, 8); HugeArray <int> b = new HugeArray <int>(8, 111, 22, 345, 40888, 3, 777, 99, 6); a.VectorMultiply(2); Assert.Equal(2u, a[0]); Assert.Equal(8u, a[3]); Assert.Equal(16u, a[7]); b.VectorMultiply(6); Assert.Equal(666, b[0]); }
/// <summary> /// Initializes the reverse index. /// </summary> private void BuildReverse() { _reverseDirectNeighbours = new HugeArray <uint>(_graph.VertexCount + 1); _additionalReverseNeighbours = new Dictionary <uint, uint[]>(); for (uint currentVertex = 1; currentVertex <= _graph.VertexCount; currentVertex++) { var edges = _graph.GetEdges(currentVertex); while (edges.MoveNext()) { uint neighbour = edges.Neighbour; this.AddReverse(neighbour, currentVertex); } } }
/// <summary> /// Copies all data from the given graph. /// </summary> /// <typeparam name="TEdgeData"></typeparam> public static void SortHilbert <TEdgeData>(this GraphBase <TEdgeData> graph, int n, Action <uint, uint> transform) where TEdgeData : struct, IGraphEdgeData { // build ranks. var ranks = graph.BuildHilbertRank(n); // invert ranks. var transformations = new HugeArray <uint>(ranks.Length); for (uint i = 0; i < ranks.Length; i++) { if (transform != null) { transform(ranks[i], i); } transformations[ranks[i]] = i; } // copy from the given graph but with sorted vertices. graph.Sort(transformations); }
/// <summary> /// Moves data around to obtain the smallest possible datastructure still containing all data. /// </summary> public void Compress() { // trim first. this.Trim(); // build a list of all vertices sorted by their first position. var sortedVertices = new HugeArray<uint>(_index.Length); for (uint i = 0; i < sortedVertices.Length; i++) { sortedVertices[i] = i; } // sort vertices and coordinates. QuickSort.Sort((i) => ((long)(_index[i] / (ulong)MAX_COLLECTION_SIZE)) * 2, (i, j) => { var tempRef = sortedVertices[i]; sortedVertices[i] = sortedVertices[j]; sortedVertices[j] = tempRef; }, 0, _index.Length - 1); // move data down. uint pointer = 0; for (uint i = 0; i < sortedVertices.Length; i++) { long index, size; if(this.TryGetIndexAndSize(sortedVertices[i], out index, out size)) { if(size > 0) { var newIndex = pointer / 2; for(var c = 0; c < size; c++) { _coordinates[pointer] = _coordinates[(index + c) * 2]; pointer++; _coordinates[pointer] = _coordinates[(index + c) * 2 + 1]; pointer++; } _index[sortedVertices[i]] = (ulong)(newIndex * MAX_COLLECTION_SIZE) + (ulong)size; } } } _nextIdx = pointer / 2; // trim again, something may have changed. this.Trim(); }
/// <summary> /// Deserializes a graph from the given stream. /// </summary> /// <param name="stream">The stream to read from. Reading will start at position 0.</param> /// <param name="edgeDataSize">The edge data size.</param> /// <param name="mapFrom">The map from for the edge data.</param> /// <param name="mapTo">The map to for the edge data.</param> /// <param name="copy">Flag to make an in-memory copy.</param> /// <returns></returns> public new static DirectedGraph <TEdgeData> Deserialize(System.IO.Stream stream, int edgeDataSize, MappedHugeArray <TEdgeData, uint> .MapFrom mapFrom, MappedHugeArray <TEdgeData, uint> .MapTo mapTo, bool copy) { // read sizes. long position = 0; stream.Seek(4, System.IO.SeekOrigin.Begin); position = position + 4; var longBytes = new byte[8]; stream.Read(longBytes, 0, 8); position = position + 8; var vertexLength = BitConverter.ToInt64(longBytes, 0); stream.Read(longBytes, 0, 8); position = position + 8; var edgeLength = BitConverter.ToInt64(longBytes, 0); var bufferSize = 32; var cacheSize = MemoryMappedHugeArrayUInt32.DefaultCacheSize; var file = new MemoryMappedStream(new OsmSharp.IO.LimitedStream(stream)); var vertexArray = new MemoryMappedHugeArrayUInt32(file, (vertexLength + 1) * VERTEX_SIZE, (vertexLength + 1) * VERTEX_SIZE, bufferSize / 4, cacheSize * 4); position = position + ((vertexLength + 1) * VERTEX_SIZE * 4); var vertexCoordinateArray = new MappedHugeArray <GeoCoordinateSimple, float>( new MemoryMappedHugeArraySingle(file, (vertexLength + 1) * 2, (vertexLength + 1) * 2, bufferSize / 4, cacheSize * 4), 2, (array, idx, coordinate) => { array[idx] = coordinate.Latitude; array[idx + 1] = coordinate.Longitude; }, (Array, idx) => { return(new GeoCoordinateSimple() { Latitude = Array[idx], Longitude = Array[idx + 1] }); }); position = position + ((vertexLength + 1) * 2 * 4); var edgeArray = new MemoryMappedHugeArrayUInt32(file, edgeLength * EDGE_SIZE, edgeLength * EDGE_SIZE, bufferSize / 2, cacheSize * 4); position = position + (edgeLength * EDGE_SIZE * 4); var edgeDataArray = new MappedHugeArray <TEdgeData, uint>( new MemoryMappedHugeArrayUInt32(file, edgeLength * edgeDataSize, edgeLength * edgeDataSize, bufferSize * 2, cacheSize * 2), edgeDataSize, mapTo, mapFrom); position = position + (edgeLength * edgeDataSize * 4); // deserialize shapes. stream.Seek(position, System.IO.SeekOrigin.Begin); var cappedStream = new OsmSharp.IO.LimitedStream(stream); var shapes = HugeCoordinateCollectionIndex.Deserialize(cappedStream, copy); if (copy) { // copy the data. var vertexArrayCopy = new HugeArray <uint>(vertexArray.Length); vertexArrayCopy.CopyFrom(vertexArray); var vertexCoordinateArrayCopy = new HugeArray <GeoCoordinateSimple>(vertexCoordinateArray.Length); vertexCoordinateArrayCopy.CopyFrom(vertexCoordinateArray); var edgeArrayCopy = new HugeArray <uint>(edgeArray.Length); edgeArrayCopy.CopyFrom(edgeArray); var edgeDataArrayCopy = new HugeArray <TEdgeData>(edgeDataArray.Length); edgeDataArrayCopy.CopyFrom(edgeDataArray); file.Dispose(); return(new DirectedGraph <TEdgeData>(vertexCoordinateArrayCopy, vertexArrayCopy, edgeArrayCopy, edgeDataArrayCopy, shapes)); } return(new DirectedGraph <TEdgeData>(vertexCoordinateArray, vertexArray, edgeArray, edgeDataArray, shapes)); }
public void HugeArraySimpleTest() { var stringArrayRef = new string[1000]; var stringArray = new HugeArray<string>(1000, 300); var randomGenerator = new RandomGenerator(66707770); // make this deterministic for (int idx = 0; idx < 1000; idx++) { if (randomGenerator.Generate(2.0) > 1) { // add data. stringArrayRef[idx] = idx.ToString(); stringArray[idx] = idx.ToString(); } else { stringArrayRef[idx] = null; stringArray[idx] = null; } } for (int idx = 0; idx < 1000; idx++) { Assert.AreEqual(stringArrayRef[idx], stringArray[idx]); } }
public void Init() { hg = new HugeArray <long>(40); }
/// <summary> /// Deserializes a huge collection index from the given stream. /// </summary> /// <param name="stream">The source stream.</param> /// <param name="copy">The copy flag. When true all data is copied into memory, otherwise the source-stream is used as a memorymapped file.</param> /// <returns></returns> public static HugeCoordinateCollectionIndex Deserialize(Stream stream, bool copy = false) { // read sizes. long position = 0; var longBytes = new byte[8]; stream.Read(longBytes, 0, 8); position = position + 8; var indexLength = BitConverter.ToInt64(longBytes, 0); stream.Read(longBytes, 0, 8); position = position + 8; var coordinateLength = BitConverter.ToInt64(longBytes, 0); var file = new MemoryMappedStream(new LimitedStream(stream)); var indexArray = new MemoryMappedHugeArrayUInt64(file, indexLength, indexLength, 1024); var coordinateArray = new MemoryMappedHugeArraySingle(file, coordinateLength * 2, coordinateLength * 2, 1024); if (copy) { // copy the data. var indexArrayCopy = new HugeArray<ulong>(indexLength); indexArrayCopy.CopyFrom(indexArray); var coordinateArrayCopy = new HugeArray<float>(coordinateLength * 2); coordinateArrayCopy.CopyFrom(coordinateArray); file.Dispose(); return new HugeCoordinateCollectionIndex(indexLength, indexArrayCopy, coordinateArrayCopy); } return new HugeCoordinateCollectionIndex(indexLength, indexArray, coordinateArray); }
public void CreateHugeNativeArray() { HugeArray <T> array = new HugeArray <T>(ArraySize); array.Close(); }