public void ForEach(Action <int, int, int, int[]> perVertexLogic) { var metadata = UnbufferedIO.ReadFile <int>(this.filePrefix + "-nodes"); var maxDegree = 0; for (int i = 0; i < metadata.Length; i += 2) { maxDegree = Math.Max(maxDegree, metadata[i + 1]); } var neighbors = new int[2 * maxDegree]; var neighborsOffset = 0; using (var reader = new UnbufferedIO.SequentialReader(this.filePrefix + "-edges")) { reader.Read(neighbors, 0, neighbors.Length); for (int i = 0; i < metadata.Length; i += 2) { var vertex = metadata[i + 0]; var degree = metadata[i + 1]; if (neighborsOffset + degree > neighbors.Length - 1) { Array.Copy(neighbors, neighborsOffset, neighbors, 0, neighbors.Length - neighborsOffset); reader.Read(neighbors, neighbors.Length - neighborsOffset, neighborsOffset); neighborsOffset = 0; } perVertexLogic(vertex, degree, neighborsOffset, neighbors); neighborsOffset += degree; } } }
public void ForEach(Action <int, int> perVertexLogic) { var metadata = UnbufferedIO.ReadFile <int>(this.filePrefix + "-nodes"); for (int i = 0; i < metadata.Length; i += 2) { perVertexLogic(metadata[i], metadata[i + 1]); } }
public static void ScanGraph(string filename, Action <int, int> perVertexLogic) { var metadata = UnbufferedIO.ReadFile <int>(filename + "-nodes"); for (int i = 0; i < metadata.Length; i += 2) { perVertexLogic(metadata[i], metadata[i + 1]); } }
public static void TestTransformedAlt(string prefix) { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var high32MetaData = UnbufferedIO.ReadFile <uint>(prefix + "-high32"); var high32Counts = 0L; for (int i = 0; i < high32MetaData.Length; i += 2) { high32Counts += high32MetaData[i + 1]; } Console.WriteLine("Read {0} high32, expecting {1} next16", high32MetaData.Length / 2, high32Counts); var next16Total = 0L; var next16Count = 0L; var next16Buffer = new ushort[1 << 20]; using (var next16Reader = new UnbufferedIO.SequentialReader(prefix + "-next16")) { for (var read = next16Reader.Read <ushort>(next16Buffer, 0, 1 << 20); read > 0; read = next16Reader.Read <ushort>(next16Buffer, 0, 1 << 20)) { next16Total += read / 2; for (int i = 0; i < read; i += 2) { next16Count += next16Buffer[i + 1]; } } } Console.WriteLine("Read {0} next16, expecting {1} next08", next16Total, next16Count); var next08Total = 0L; var next08Count = 0L; var next08Buffer = new byte[1 << 20]; using (var next08Reader = new UnbufferedIO.SequentialReader(prefix + "-next08")) { for (var read = next08Reader.Read <byte>(next08Buffer, 0, 1 << 20); read > 0; read = next08Reader.Read <byte>(next08Buffer, 0, 1 << 20)) { next08Total += read / 2; for (int i = 0; i < read; i += 2) { next08Count += next08Buffer[i + 1] == 0 ? 256 : (int)next08Buffer[i + 1]; } } } Console.WriteLine("Read {0} next08, expecting {1} last08", next08Total, next08Count); Console.WriteLine(stopwatch.Elapsed); }
public static unsafe void HilbertPagerank(string filename, float *a, float *b, uint nodes, float reset) { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var degrees = LargePages.AllocateInts(nodes); //var degrees = new int[nodes]; var metadata = UnbufferedIO.ReadFile <uint>(filename + "-upper"); var edges = UnbufferedIO.ReadFile <uint>(filename + "-lower"); Console.WriteLine("{2}\tRead header of {0} blocks, for {1} edges", metadata.Length / 3, edges.Length, stopwatch.Elapsed); ScanGraphHilbert(metadata, edges, (srcUpper, tgtUpper, offset, count, edgeArray) => { for (var j = offset; j < offset + count; j++) { degrees[srcUpper + (edges[j] & 0xFFFF)]++; } }); Console.WriteLine("{0}\tDegrees calculated", stopwatch.Elapsed); for (int iteration = 0; iteration < 20; iteration++) { var startTime = stopwatch.Elapsed; for (int i = 0; i < nodes; i++) { b[i] = (1.0f - reset) * a[i] / degrees[i]; a[i] = reset; } ScanGraphHilbert(metadata, edges, (sourceUpper, targetUpper, offset, count, edgeArray) => { for (var j = offset; j < offset + count; j++) { a[targetUpper + (edges[j] >> 16)] += b[sourceUpper + (edges[j] & 0xFFFF)]; } }); Console.WriteLine("{0}\tIteration {1} in {2}", stopwatch.Elapsed, iteration, stopwatch.Elapsed - startTime); } }
public unsafe static void PageRankStep(string filename, float *a, float *b, uint nodes, float reset) { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var nodeArray = UnbufferedIO.ReadFile <int>(filename + "-nodes"); var edgeArray = UnbufferedIO.ReadFile <int>(filename + "-edges"); for (int i = 0; i < nodes; i++) { a[i] = reset; } for (int iteration = 0; iteration < 20; iteration++) { var startTime = stopwatch.Elapsed; for (int i = 0; i < nodes; i++) { b[i] = a[i]; a[i] = 0.0f; } // apply per-vertex pagerank logic across the graph ScanGraph(nodeArray, edgeArray, (vertex, degree, offset, neighbors) => { if (b[vertex] > 0.0f) { var update = (1.0f - reset) * b[vertex] / degree; for (int i = 0; i < degree; i++) { a[neighbors[offset + i]] += update; } } }); Console.WriteLine("{0}\tIteration {1}", stopwatch.Elapsed - startTime, iteration); } Console.WriteLine(stopwatch.Elapsed); }
public MultiHilbertScanner(string prefix) { this.prefix = prefix; this.high32Buffer = UnbufferedIO.ReadFile <uint>(prefix + "-high32"); this.next16Buffer = UnbufferedIO.ReadFile <ushort>(prefix + "-next16"); this.next08Buffer = UnbufferedIO.ReadFile <byte>(prefix + "-next08"); this.last08ToRead = new int[high32Buffer.Length / 2]; var next16Offset = 0; var next08Offset = 0; for (int i = 0; i < high32Buffer.Length; i += 2) { var high32value = high32Buffer[i]; var high32count = (int)high32Buffer[i + 1]; var next16count = 0; for (int j = 0; j < high32count; j++) { next16count += (int)(next16Buffer[next16Offset + 2 * j + 1] == 0 ? ((uint)ushort.MaxValue) + 1 : next16Buffer[next16Offset + 2 * j + 1]); } next16Offset += 2 * high32count; var next08count = 0; for (int k = 0; k < next16count; k++) { next08count += (int)(next08Buffer[next08Offset + 2 * k + 1] == 0 ? ((uint)byte.MaxValue) + 1 : next08Buffer[next08Offset + 2 * k + 1]); } next08Offset += 2 * next16count; last08ToRead[i / 2] = next08count; } this.last08Buffer = new byte[last08ToRead.Max()]; }
public static void ScanGraphHilbert(string filename, Action <uint, uint, uint, uint, uint[]> perVertexLogic) { var metadata = UnbufferedIO.ReadFile <uint>(filename + "-upper"); var maxCount = (uint)0; for (int i = 0; i < metadata.Length; i += 3) { maxCount = Math.Max(maxCount, metadata[i + 2]); } var edges = new uint[2 * maxCount]; var edgesOffset = (uint)0; using (var reader = new UnbufferedIO.SequentialReader(filename + "-lower")) { reader.Read(edges, 0, edges.Length); for (int i = 0; i < metadata.Length; i += 3) { var sourceUpper = metadata[i + 0] << 16; var targetUpper = metadata[i + 1] << 16; var count = metadata[i + 2]; if (edgesOffset + count > edges.Length - 1) { Array.Copy(edges, edgesOffset, edges, 0, edges.Length - edgesOffset); reader.Read(edges, (int)(edges.Length - edgesOffset), (int)edgesOffset); edgesOffset = 0; } perVertexLogic(sourceUpper, targetUpper, edgesOffset, count, edges); edgesOffset += count; } } }
public static void TestTransformed(string prefix) { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var high32MetaData = UnbufferedIO.ReadFile <uint>(prefix + "-high32"); var next16Reader = new UnbufferedIO.SequentialReader(prefix + "-next16"); var next08Reader = new UnbufferedIO.SequentialReader(prefix + "-next08"); var last08Reader = new UnbufferedIO.SequentialReader(prefix + "-last08"); var next16Buffer = new ushort[] { }; var next08Buffer = new byte[] { }; var last08Buffer = new byte[] { }; var iDegrees = new uint[106000000]; var oDegrees = new uint[106000000]; var edgeCount = 0L; for (int i = 0; i < high32MetaData.Length; i += 2) { var high32value = high32MetaData[i]; var high32count = (int)high32MetaData[i + 1]; if (next16Buffer.Length < 2 * high32count) { next16Buffer = new ushort[2 * high32count]; } var readNext16 = next16Reader.Read <ushort>(next16Buffer, 0, 2 * high32count); if (readNext16 != 2 * high32count) { Console.WriteLine("Read error, next16; requested {0}, read {1}", 2 * high32count, readNext16); } var next16count = 0; for (int j = 0; j < high32count; j++) { next16count += (int)(next16Buffer[2 * j + 1] == 0 ? ((uint)ushort.MaxValue) + 1 : next16Buffer[2 * j + 1]); } if (next08Buffer.Length < 2 * next16count) { next08Buffer = new byte[2 * next16count]; } var readNext08 = next08Reader.Read <byte>(next08Buffer, 0, 2 * next16count); if (readNext08 != 2 * next16count) { Console.WriteLine("Read error, next08; requested {0}, read {1}", 2 * next16count, readNext08); } var next08count = 0; for (int k = 0; k < next16count; k++) { next08count += (int)(next08Buffer[2 * k + 1] == 0 ? ((uint)byte.MaxValue) + 1 : next08Buffer[2 * k + 1]); } if (last08Buffer.Length < next08count) { last08Buffer = new byte[next08count]; } var readLast08 = last08Reader.Read <byte>(last08Buffer, 0, next08count); if (readLast08 != next08count) { Console.WriteLine("Read error, last08; requested {0}, read {1}", next08count, readLast08); } var next16Offset = 0; var next08Offset = 0; var last08Offset = 0; // do work for all these dudes var xHigh32 = (high32value & 0xFFFF0000) << 0; var yHigh32 = (high32value & 0x0000FFFF) << 16; for (int j = 0; j < high32count; j++) { var xNext16 = xHigh32 + ((next16Buffer[next16Offset + 2 * j] & 0xFF00) << 0); var yNext16 = yHigh32 + ((next16Buffer[next16Offset + 2 * j] & 0x00FF) << 8); next16count = next16Buffer[next16Offset + 2 * j + 1]; if (next16count == 0) { next16count = ((int)ushort.MaxValue) + 1; } for (int k = 0; k < next16count; k++) { var xNext08 = xNext16 + ((next08Buffer[next08Offset + 2 * k] & 0xF0) << 0); var yNext08 = yNext16 + ((next08Buffer[next08Offset + 2 * k] & 0x0F) << 4); next08count = next08Buffer[next08Offset + 2 * k + 1]; if (next08count == 0) { next08count = ((int)byte.MaxValue) + 1; } for (int l = 0; l < next08count; l++) { var x = xNext08 + ((last08Buffer[last08Offset + l] & 0xF0) >> 4); var y = yNext08 + ((last08Buffer[last08Offset + l] & 0x0F) >> 0); iDegrees[y]++; oDegrees[x]++; edgeCount++; } last08Offset += next08count; } next08Offset += 2 * next16count; } } Console.WriteLine("{1}\tProcessed\t{0} edges", edgeCount, stopwatch.Elapsed); next16Reader.Dispose(); next08Reader.Dispose(); last08Reader.Dispose(); var graph = new FileGraphScanner(prefix); graph.ForEach((vertex, degree, offset, neighbors) => { for (int i = 0; i < degree; i++) { iDegrees[neighbors[offset + i]]--; oDegrees[vertex]--; } }); var iErrors = iDegrees.Count(x => x > 0); var oErrors = oDegrees.Count(x => x > 0); Console.WriteLine("iErrors: {0}\toErrors: {1}", iErrors, oErrors); Console.WriteLine("{0}\tCounted {1} edges", stopwatch.Elapsed, edgeCount); }
public static void ScanAlt(this string prefix, Action <uint, uint, int, int, byte[]> perEdgeAction) { var high32Buffer = UnbufferedIO.ReadFile <uint>(prefix + "-high32"); var next16Buffer = UnbufferedIO.ReadFile <ushort>(prefix + "-next16"); var next08Buffer = UnbufferedIO.ReadFile <byte>(prefix + "-next08"); var last08ToRead = new int[high32Buffer.Length / 2]; var next16Offset = 0; var next08Offset = 0; for (int i = 0; i < high32Buffer.Length; i += 2) { var high32value = high32Buffer[i]; var high32count = (int)high32Buffer[i + 1]; var next16count = 0; for (int j = 0; j < high32count; j++) { next16count += (int)(next16Buffer[next16Offset + 2 * j + 1] == 0 ? ((uint)ushort.MaxValue) + 1 : next16Buffer[next16Offset + 2 * j + 1]); } next16Offset += 2 * high32count; var next08count = 0; for (int k = 0; k < next16count; k++) { next08count += (int)(next08Buffer[next08Offset + 2 * k + 1] == 0 ? ((uint)byte.MaxValue) + 1 : next08Buffer[next08Offset + 2 * k + 1]); } next08Offset += 2 * next16count; last08ToRead[i / 2] = next08count; } next16Offset = 0; next08Offset = 0; var last08Reader = new UnbufferedIO.SequentialReader(prefix + "-last08"); var last08Buffer = new byte[last08ToRead.Max()]; for (int i = 0; i < high32Buffer.Length; i += 2) { var high32value = high32Buffer[i]; var high32count = (int)high32Buffer[i + 1]; var readLast08 = last08Reader.Read <byte>(last08Buffer, 0, last08ToRead[i / 2]); if (readLast08 != last08ToRead[i / 2]) { Console.WriteLine("Read error, last08; requested {0}, read {1}", last08ToRead[i / 2], readLast08); } var last08Offset = 0; // do work for all these dudes var xHigh32 = (high32value & 0xFFFF0000) << 0; var yHigh32 = (high32value & 0x0000FFFF) << 16; for (int j = 0; j < high32count; j++) { var xNext16 = xHigh32 + (uint)((next16Buffer[next16Offset + 2 * j] & 0xFF00) << 0); var yNext16 = yHigh32 + (uint)((next16Buffer[next16Offset + 2 * j] & 0x00FF) << 8); var next16count = (int)next16Buffer[next16Offset + 2 * j + 1]; if (next16count == 0) { next16count = ((int)ushort.MaxValue) + 1; } for (int k = 0; k < next16count; k++) { var xNext08 = xNext16 + (uint)((next08Buffer[next08Offset + 2 * k] & 0xF0) << 0); var yNext08 = yNext16 + (uint)((next08Buffer[next08Offset + 2 * k] & 0x0F) << 4); var next08count = (int)next08Buffer[next08Offset + 2 * k + 1]; if (next08count == 0) { next08count = ((int)byte.MaxValue) + 1; } perEdgeAction(xNext08, yNext08, next08count, last08Offset, last08Buffer); last08Offset += next08count; } next08Offset += 2 * next16count; } next16Offset += 2 * high32count; } last08Reader.Dispose(); }
public MemoryGraphScanner(string prefix) { this.metadata = UnbufferedIO.ReadFile <int>(prefix + "-nodes"); this.neighbors = UnbufferedIO.ReadFile <int>(prefix + "-edges"); }