public unsafe static void ClumsyCC(string filename, uint nodes) { var labels = LargePages.AllocateInts(nodes); for (int i = 0; i < nodes; i++) { labels[i] = i; } var oldSum = Int64.MaxValue; var newSum = 0L; for (int i = 0; i < nodes; i++) { newSum += labels[i]; } var stopwatch = System.Diagnostics.Stopwatch.StartNew(); while (newSum < oldSum) { var startTime = stopwatch.Elapsed; // apply per-vertex pagerank logic across the graph ScanGraph(filename, (vertex, degree, offset, neighbors) => { var label = labels[vertex]; for (int i = 0; i < degree; i++) { label = Math.Min(label, labels[neighbors[offset + i]]); } labels[vertex] = label; for (int i = 0; i < degree; i++) { labels[neighbors[offset + i]] = label; } }); oldSum = newSum; newSum = 0L; for (int i = 0; i < nodes; i++) { newSum += labels[i]; } Console.WriteLine("{0}", stopwatch.Elapsed - startTime); } }
public static unsafe void HilbertPagerank(string filename, float *a, float *b, uint nodes, float reset) { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var degrees = LargePages.AllocateInts(nodes); //var degrees = new int[nodes]; var metadata = UnbufferedIO.ReadFile <uint>(filename + "-upper"); var edges = UnbufferedIO.ReadFile <uint>(filename + "-lower"); Console.WriteLine("{2}\tRead header of {0} blocks, for {1} edges", metadata.Length / 3, edges.Length, stopwatch.Elapsed); ScanGraphHilbert(metadata, edges, (srcUpper, tgtUpper, offset, count, edgeArray) => { for (var j = offset; j < offset + count; j++) { degrees[srcUpper + (edges[j] & 0xFFFF)]++; } }); Console.WriteLine("{0}\tDegrees calculated", stopwatch.Elapsed); for (int iteration = 0; iteration < 20; iteration++) { var startTime = stopwatch.Elapsed; for (int i = 0; i < nodes; i++) { b[i] = (1.0f - reset) * a[i] / degrees[i]; a[i] = reset; } ScanGraphHilbert(metadata, edges, (sourceUpper, targetUpper, offset, count, edgeArray) => { for (var j = offset; j < offset + count; j++) { a[targetUpper + (edges[j] >> 16)] += b[sourceUpper + (edges[j] & 0xFFFF)]; } }); Console.WriteLine("{0}\tIteration {1} in {2}", stopwatch.Elapsed, iteration, stopwatch.Elapsed - startTime); } }
public static unsafe void HilbertPagerankFromDisk(string filename, float[] a, float[] b, uint nodes, float reset) { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); #if UseLargePages var degrees = LargePages.AllocateInts(nodes); #else var degrees = new int[nodes]; #endif ScanGraphHilbert(filename, (srcUpper, tgtUpper, offset, count, edges) => { for (int i = 0; i < count; i++) { degrees[srcUpper + (edges[offset + i] >> 16)]++; } }); Console.WriteLine("{0}\tDegrees calculated", stopwatch.Elapsed); for (int iteration = 0; iteration < 20; iteration++) { var startTime = stopwatch.Elapsed; ScanGraphHilbert(filename, (srcUpper, tgtUpper, offset, count, edges) => { for (int i = 0; i < count; i++) { a[tgtUpper + (edges[offset + i] >> 16)] += b[srcUpper + (edges[offset + i] & 0xFFFF)]; } }); for (int i = 0; i < nodes; i++) { a[i] = a[i] / degrees[i]; } Console.WriteLine("{0}\tIteration {1} in {2}", stopwatch.Elapsed, iteration, stopwatch.Elapsed - startTime); } }
public static unsafe void MultiHilbertPagerank(string filename, float *a, float *b, uint nodes, float reset) { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var degrees = LargePages.AllocateInts(nodes); var scanner = new MultiHilbertScanner(filename); scanner.Scan((xUpper, yUpper, count, offset, edges) => { for (var j = offset; j < offset + count; j++) { degrees[xUpper + ((edges[j] & 0xF0) >> 4)]++; } }); Console.WriteLine("{0}\tDegrees calculated", stopwatch.Elapsed); for (int iteration = 0; iteration < 20; iteration++) { var startTime = stopwatch.Elapsed; for (int i = 0; i < nodes; i++) { b[i] = (1.0f - reset) * a[i] / degrees[i]; a[i] = reset; } scanner.Scan((xUpper, yUpper, count, offset, edges) => { for (var j = offset; j < offset + count; j++) { a[yUpper + (edges[j] & 0x0F)] += b[xUpper + ((edges[j] & 0xF0) >> 4)]; } }); Console.WriteLine("{0}\tIteration {1} in {2}", stopwatch.Elapsed, iteration, stopwatch.Elapsed - startTime); } }
public static unsafe void HilbertUnionFind2(string filename, uint nodes) { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); #if UseLargePages var roots = LargePages.AllocateInts(nodes); var ranks = LargePages.AllocateBytes(nodes); #else var roots = new int[nodes]; var ranks = new byte[nodes]; #endif for (int i = 0; i < nodes; i++) { roots[i] = i; } ScanGraphHilbert(filename, (sourceUpper, targetUpper, offset, count, edges) => { for (int j = 0; j < count; j++) { var source = (int)(sourceUpper + (edges[offset + j] & 0xFFFF)); var target = (int)(targetUpper + (edges[offset + j] >> 16)); if (source != target) { while (source != roots[source]) { source = roots[source]; } while (target != roots[target]) { target = roots[target]; } // union(source, target) if (source != target) { // there may be a tie in ranks if (ranks[source] == ranks[target]) { // break ties towards lower ids if (source < target) { ranks[source]++; roots[target] = source; } else { ranks[target]++; roots[source] = target; } } else { // attatch lower rank to higher if (ranks[source] < ranks[target]) { roots[source] = target; } else { roots[target] = source; } } } } } }); // path compress all vertices to roots. for (int i = 0; i < nodes; i++) { while (roots[i] != roots[roots[i]]) { roots[i] = roots[roots[i]]; } } var counter = 0; for (int i = 0; i < nodes; i++) { if (roots[i] != i) { counter++; } } Console.WriteLine("Edges found: {0}", counter); }
public static unsafe void HilbertUnionFind(string filenameUpper, string filenameLower, uint nodes) { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); using (var upper = System.IO.File.OpenRead(filenameUpper)) { using (var lower = System.IO.File.OpenRead(filenameLower)) { var bytes = new byte[upper.Length]; upper.Read(bytes, 0, bytes.Length); var uppers = new int[bytes.Length / 4]; Buffer.BlockCopy(bytes, 0, uppers, 0, bytes.Length); var maxEdges = 0; for (int i = 0; i < uppers.Length; i += 3) { maxEdges = Math.Max(maxEdges, uppers[i + 2]); } Console.WriteLine("{1}\tRead header of {0} blocks", uppers.Length / 3, stopwatch.Elapsed); bytes = new byte[8 * maxEdges]; var lowers = new UInt16[2 * maxEdges]; var edges = new Int32[2 * maxEdges]; #if UseLargePages var roots = LargePages.AllocateInts(nodes); var ranks = LargePages.AllocateBytes(nodes); #else var roots = new int[nodes]; var ranks = new byte[nodes]; #endif for (int i = 0; i < nodes; i++) { roots[i] = i; } for (int i = 0; i < uppers.Length; i += 3) { var read = lower.Read(bytes, 0, 4 * uppers[i + 2]); Buffer.BlockCopy(bytes, 0, lowers, 0, read); var sourceUpper = uppers[i + 0] << 16; var targetUpper = uppers[i + 1] << 16; var count = uppers[i + 2]; for (int j = 0; j < count; j++) { edges[2 * j + 0] = roots[sourceUpper + lowers[2 * j + 0]]; edges[2 * j + 1] = roots[targetUpper + lowers[2 * j + 1]]; } for (int j = 0; j < count; j++) { var source = edges[2 * j + 0]; var target = edges[2 * j + 1]; if (source != target) { while (source != roots[source]) { source = roots[source]; } while (target != roots[target]) { target = roots[target]; } // union(source, target) if (source != target) { // there may be a tie in ranks if (ranks[source] == ranks[target]) { // break ties towards lower ids if (source < target) { ranks[source]++; roots[target] = source; } else { ranks[target]++; roots[source] = target; } } else { // attatch lower rank to higher if (ranks[source] < ranks[target]) { roots[source] = target; } else { roots[target] = source; } } } } } } // path compress all vertices to roots. for (int i = 0; i < nodes; i++) { while (roots[i] != roots[roots[i]]) { roots[i] = roots[roots[i]]; } } var counter = 0; for (int i = 0; i < nodes; i++) { if (roots[i] != i) { counter++; } } Console.WriteLine("Edges found: {0}", counter); } } }
public static unsafe void MultiHilbertCC(string filename, uint nodes) { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var roots = LargePages.AllocateInts(nodes); var ranks = LargePages.AllocateBytes(nodes); for (int i = 0; i < nodes; i++) { roots[i] = i; } var scanner = new MultiHilbertScanner(filename); scanner.Scan((xUpper, yUpper, count, offset, edges) => { for (int j = 0; j < count; j++) { var source = (int)(xUpper + ((edges[offset + j] & 0xF0) >> 4)); var target = (int)(yUpper + ((edges[offset + j] & 0x0F) >> 0)); if (source != target) { while (source != roots[source]) { source = roots[source]; } while (target != roots[target]) { target = roots[target]; } // union(source, target) if (source != target) { // there may be a tie in ranks if (ranks[source] == ranks[target]) { // break ties towards lower ids if (source < target) { ranks[source]++; roots[target] = source; } else { ranks[target]++; roots[source] = target; } } else { // attatch lower rank to higher if (ranks[source] < ranks[target]) { roots[source] = target; } else { roots[target] = source; } } } } } }); // path compress all vertices to roots. for (int i = 0; i < nodes; i++) { while (roots[i] != roots[roots[i]]) { roots[i] = roots[roots[i]]; } } var counter = 0; for (int i = 0; i < nodes; i++) { if (roots[i] != i) { counter++; } } Console.WriteLine("{1}\tEdges found: {0}", counter, stopwatch.Elapsed); }
public unsafe static void ConnectedComponents(string filename, uint nodes) { #if UseLargePages var roots = LargePages.AllocateInts(nodes); var ranks = LargePages.AllocateBytes(nodes); #else var roots = new int[nodes]; var ranks = new byte[nodes]; #endif for (int i = 0; i < nodes; i++) { roots[i] = i; } // apply per-vertex union-find logic across the graph ScanGraph(filename, (vertex, degree, offset, neighbors) => { // find(vertex) var source = roots[vertex]; while (source != roots[source]) { source = roots[source]; } for (int i = 0; i < degree; i++) { // find(neighbors[i]) var target = roots[neighbors[offset + i]]; while (target != roots[target]) { target = roots[target]; } // union(source, target) if (source != target) { // there may be a tie in ranks if (ranks[source] == ranks[target]) { // break ties towards lower ids if (source < target) { ranks[source]++; roots[target] = source; } else { ranks[target]++; roots[source] = target; source = target; } } else { // attatch lower rank to higher if (ranks[source] < ranks[target]) { roots[source] = target; source = target; } else { roots[target] = source; } } } } }); var counter = 0; for (int i = 0; i < nodes; i++) { if (roots[i] != i) { counter++; } } Console.WriteLine("Edges found: {0}", counter); }
static void ExecuteSingleThreaded(string[] args, string dataDir) { string ukFile = Path.Combine(dataDir, @"uk-2007-05"); string twitterFile = Path.Combine(dataDir, @"twitter_rv.bin"); string livejournalFile = Path.Combine(dataDir, @"livejournal.bin"); if (args.Length < 3) { throw new Exception("Three arguments required: system, algorithm, dataset"); } var algorithm = args[1]; var dataset = args[2]; #region file conversions if (algorithm == "convert" && dataset == "twitter") { SingleThreaded.ConvertGraph(twitterFile); } if (algorithm == "partition" && dataset == "twitter") { SingleThreaded.PartitionGraph(twitterFile, 4, (s, t) => (s & 1) + 2 * (t & 1), dataDir + @"twitter-part-{0}-of-{1}"); } if (algorithm == "transpose" && dataset == "twitter") { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); SingleThreaded.TransposeGraph(dataDir + @"twitterfollowers\twitter_rv.bin", 65000000); Console.WriteLine(stopwatch.Elapsed); } #endregion #region hilbert layout if (algorithm == "hilbertlayout" && dataset == "twitter") { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); uint nodes = 0; var names = new uint[65000000]; for (int i = 0; i < names.Length; i++) { names[i] = uint.MaxValue; } var idegrees = new int[65000000]; var odegrees = new int[65000000]; var edges = 0L; SingleThreaded.ScanGraph(twitterFile, (vertex, degree, offset, neighbors) => { if (names[vertex] == uint.MaxValue) { names[vertex] = nodes++; } for (int i = 0; i < degree; i++) { if (names[neighbors[offset + i]] == uint.MaxValue) { names[neighbors[offset + i]] = nodes++; } } edges += degree; }); Console.WriteLine("{2}\tNodes: {0}\tEdges: {1}", nodes, edges, stopwatch.Elapsed); // allocate enough space for all the edges. var hilbertTransformed = new uint[edges]; var counts = new uint[1 << 20]; SingleThreaded.ScanGraph(twitterFile, (vertex, degree, offset, neighbors) => { for (int i = 0; i < degree; i++) { counts[HilbertCurve.xy2dByte(names[vertex], names[neighbors[offset + i]]) >> 32]++; } }); Console.WriteLine("{0}\tHilbert regions sized", stopwatch.Elapsed); for (int i = 1; i < counts.Length; i++) { counts[i + 1] += counts[i]; } for (int i = counts.Length - 1; i > 0; i--) { counts[i] = counts[i] - 1; } counts[0] = 0; var Trie = new BufferTrie <uint>(20, (array, offset, length) => { for (int i = offset; i < offset + length; i++) { hilbertTransformed[counts[array[i].Index]++] = array[i].Value; } }); for (int i = counts.Length - 1; i > 0; i--) { counts[i] = counts[i] - 1; } counts[0] = 0; var buffer = new BufferTrie <uint> .Pair[5000000]; SingleThreaded.ScanGraph(twitterFile, (vertex, degree, offset, neighbors) => { for (int i = 0; i < degree; i++) { var result = HilbertCurve.xy2dByte(names[vertex], names[neighbors[offset + i]]); buffer[i] = new BufferTrie <uint> .Pair((int)(result >> 32), (uint)(result & 0xFFFF)); } Trie.Insert(buffer, 0, degree); }); Trie.Flush(); Console.WriteLine("{0}\tEdges partitioned", stopwatch.Elapsed); using (var upper = new System.IO.BinaryWriter(System.IO.File.OpenWrite("twitter-hilbert-upper"))) { for (uint i = 0; i < counts.Length - 1; i++) { if (counts[i] < counts[i + 1]) { uint x = 0, y = 0; HilbertCurve.d2xyByte((i << 32), out x, out y); upper.Write(x); upper.Write(y); upper.Write(counts[i + 1] - counts[i]); } } } using (var lower = new System.IO.BinaryWriter(System.IO.File.OpenWrite("twitter-hilbert-lower"))) { for (uint i = 0; i < counts.Length - 1; i++) { Array.Sort(hilbertTransformed, (int)counts[i], (int)(counts[i + 1] - counts[i])); for (uint j = counts[i]; j < counts[i + 1]; j++) { uint x = 0, y = 0; HilbertCurve.d2xyByte((i << 32) + hilbertTransformed[j], out x, out y); lower.Write((UInt16)(x & 0xFFFF)); lower.Write((UInt16)(y & 0xFFFF)); } } } } #endregion #region hilbert pagerank if (algorithm == "hilbertpagerank" && dataset == "livejournal") { unsafe { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var nodes = (uint)42000000; #if UseLargePages var srcRanks = LargePages.AllocateFloats(nodes); var dstRanks = LargePages.AllocateFloats(nodes); #else var srcRanks = new float[nodes]; var dstRanks = new float[nodes]; #endif for (int i = 0; i < nodes; i++) { srcRanks[i] = 1.0f; } SingleThreaded.HilbertPagerank(@"livejournal-hilbert", dstRanks, srcRanks, nodes, 0.85f); Console.WriteLine(stopwatch.Elapsed); } } if (algorithm == "hilbertpagerank" && dataset == "twitter") { unsafe { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var nodes = (uint)42000000; #if UseLargePages var srcRanks = LargePages.AllocateFloats(nodes); var dstRanks = LargePages.AllocateFloats(nodes); #else var srcRanks = new float[nodes]; var dstRanks = new float[nodes]; #endif for (int i = 0; i < nodes; i++) { srcRanks[i] = 1.0f; } SingleThreaded.HilbertPagerank(@"twitter-hilbert", dstRanks, srcRanks, nodes, 0.85f); Console.WriteLine(stopwatch.Elapsed); } } if (algorithm == "hilbertpagerank" && dataset == "uk-2007-05") { unsafe { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var nodes = (uint)106000000; var srcRanks = LargePages.AllocateFloats(nodes); var dstRanks = LargePages.AllocateFloats(nodes); for (int i = 0; i < nodes; i++) { srcRanks[i] = 1.0f; } SingleThreaded.MultiHilbertPagerank(@"uk-2007-05-hilbert", dstRanks, srcRanks, nodes, 0.85f); Console.WriteLine(stopwatch.Elapsed); } } #endregion #region hilbert union find if (algorithm == "hilbertunionfind" && dataset == "twitter") { unsafe { var nodes = (uint)42000000; var stopwatch = System.Diagnostics.Stopwatch.StartNew(); SingleThreaded.HilbertUnionFind2(@"twitter-hilbert", nodes); Console.WriteLine(stopwatch.Elapsed); } } if (algorithm == "hilbertunionfind" && dataset == "uk-2007-05") { unsafe { var nodes = (uint)106000000; var stopwatch = System.Diagnostics.Stopwatch.StartNew(); SingleThreaded.MultiHilbertCC(@"uk-2007-05-hilbert", nodes); Console.WriteLine(stopwatch.Elapsed); } } if (algorithm == "hilbertunionfind" && dataset == "livejournal") { unsafe { var nodes = (uint)42000000; var stopwatch = System.Diagnostics.Stopwatch.StartNew(); SingleThreaded.HilbertUnionFind(@"livejournal-hilbert-upper", "livejournal-hilbert-lower", nodes); Console.WriteLine(stopwatch.Elapsed); } } #endregion #region page rank if (algorithm == "pagerank" && dataset == "uk-2007-05") { unsafe { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var nodes = (uint)106000000; #if UseLargePages var srcRanks = LargePages.AllocateFloats(nodes); var dstRanks = LargePages.AllocateFloats(nodes); #else var srcRanks = new float[nodes]; var dstRanks = new float[nodes]; #endif SingleThreaded.PageRankFromDisk(ukFile, dstRanks, srcRanks, nodes, 0.85f); Console.WriteLine(stopwatch.Elapsed); } } if (algorithm == "pagerank" && dataset == "twitter") { unsafe { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var nodes = (uint)65000000; #if UseLargePages var srcRanks = LargePages.AllocateFloats(nodes); var dstRanks = LargePages.AllocateFloats(nodes); #else var srcRanks = new float[nodes]; var dstRanks = new float[nodes]; #endif SingleThreaded.PageRankFromDisk(twitterFile, dstRanks, srcRanks, nodes, 0.85f); Console.WriteLine(stopwatch.Elapsed); } } if (algorithm == "pagerank" && dataset == "livejournal") { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); unsafe { var nodes = (uint)65000000; #if UseLargePages var srcRanks = LargePages.AllocateFloats(nodes); var dstRanks = LargePages.AllocateFloats(nodes); #else var srcRanks = new float[nodes]; var dstRanks = new float[nodes]; #endif for (int i = 0; i < 20; i++) { SingleThreaded.PageRankStep(livejournalFile, dstRanks, srcRanks, nodes, 0.85f); Console.WriteLine("{0}\tIteration {1}", stopwatch.Elapsed, i); } } Console.WriteLine(stopwatch.Elapsed); } #endregion #region connected components if (algorithm == "connectedcomponents" && dataset == "twitter") { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); SingleThreaded.ConnectedComponents(twitterFile, 65000000); Console.WriteLine(stopwatch.Elapsed); } if (algorithm == "connectedcomponents" && dataset == "uk-2007-05") { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); SingleThreaded.ClumsyCC(ukFile, 106000000); Console.WriteLine(stopwatch.Elapsed); } if (algorithm == "connectedcomponents" && dataset == "livejournal") { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); SingleThreaded.ConnectedComponents(livejournalFile, 6500000); Console.WriteLine(stopwatch.Elapsed); } #endregion #region maximal independent set if (algorithm == "maximalindependentset" && dataset == "twitter") { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); SingleThreaded.MaximalIndependentSet(twitterFile, 65000000); Console.WriteLine(stopwatch.Elapsed); } if (algorithm == "maximalindependentset" && dataset == "livejournal") { var stopwatch = System.Diagnostics.Stopwatch.StartNew(); SingleThreaded.MaximalIndependentSet(livejournalFile, 6500000); Console.WriteLine(stopwatch.Elapsed); } #endregion }