public static void TransposeGraph(string filename, int nodecount)
        {
            var nodes = new int[nodecount + 1];

            ScanGraph(filename, (vertex, degree, offset, neighbors) =>
            {
                for (int i = 0; i < degree; i++)
                {
                    nodes[neighbors[offset + i]]++;
                }
            });

            // accumulate to determine ending offsets
            for (int i = 1; i < nodes.Length; i++)
            {
                nodes[i] += nodes[i - 1];
            }

            // shift to determine starting offsets
            for (int i = nodes.Length - 1; i > 0; i--)
            {
                nodes[i] = nodes[i - 1];
            }

            nodes[0] = 0;

            var edges = new int[nodes[nodecount - 1]];

            var buffer = new BufferTrie <int> .Pair[5000000];

            var edgeTrie = new BufferTrie <int>(26, (array, offset, length) =>
            {
                for (int i = 0; i < length; i++)
                {
                    edges[nodes[array[i].Index]++] = array[i].Value;
                }
            });


            ScanGraph(filename, (vertex, degree, offset, neighbors) =>
            {
                for (int i = 0; i < degree; i++)
                {
                    buffer[i] = new BufferTrie <int> .Pair(neighbors[offset + i], vertex);
                }

                edgeTrie.Insert(buffer, 0, degree);
            });

            edgeTrie.Flush();
        }
Beispiel #2
0
        static void ExecuteSingleThreaded(string[] args, string dataDir)
        {
            string ukFile          = Path.Combine(dataDir, @"uk-2007-05");
            string twitterFile     = Path.Combine(dataDir, @"twitter_rv.bin");
            string livejournalFile = Path.Combine(dataDir, @"livejournal.bin");

            if (args.Length < 3)
            {
                throw new Exception("Three arguments required: system, algorithm, dataset");
            }

            var algorithm = args[1];
            var dataset   = args[2];

            #region file conversions
            if (algorithm == "convert" && dataset == "twitter")
            {
                SingleThreaded.ConvertGraph(twitterFile);
            }

            if (algorithm == "partition" && dataset == "twitter")
            {
                SingleThreaded.PartitionGraph(twitterFile, 4, (s, t) => (s & 1) + 2 * (t & 1), dataDir + @"twitter-part-{0}-of-{1}");
            }

            if (algorithm == "transpose" && dataset == "twitter")
            {
                var stopwatch = System.Diagnostics.Stopwatch.StartNew();
                SingleThreaded.TransposeGraph(dataDir + @"twitterfollowers\twitter_rv.bin", 65000000);
                Console.WriteLine(stopwatch.Elapsed);
            }
            #endregion

            #region hilbert layout
            if (algorithm == "hilbertlayout" && dataset == "twitter")
            {
                var stopwatch = System.Diagnostics.Stopwatch.StartNew();

                uint nodes = 0;
                var  names = new uint[65000000];
                for (int i = 0; i < names.Length; i++)
                {
                    names[i] = uint.MaxValue;
                }

                var idegrees = new int[65000000];
                var odegrees = new int[65000000];

                var edges = 0L;
                SingleThreaded.ScanGraph(twitterFile, (vertex, degree, offset, neighbors) =>
                {
                    if (names[vertex] == uint.MaxValue)
                    {
                        names[vertex] = nodes++;
                    }

                    for (int i = 0; i < degree; i++)
                    {
                        if (names[neighbors[offset + i]] == uint.MaxValue)
                        {
                            names[neighbors[offset + i]] = nodes++;
                        }
                    }

                    edges += degree;
                });

                Console.WriteLine("{2}\tNodes: {0}\tEdges: {1}", nodes, edges, stopwatch.Elapsed);

                // allocate enough space for all the edges.
                var hilbertTransformed = new uint[edges];

                var counts = new uint[1 << 20];
                SingleThreaded.ScanGraph(twitterFile, (vertex, degree, offset, neighbors) =>
                {
                    for (int i = 0; i < degree; i++)
                    {
                        counts[HilbertCurve.xy2dByte(names[vertex], names[neighbors[offset + i]]) >> 32]++;
                    }
                });

                Console.WriteLine("{0}\tHilbert regions sized", stopwatch.Elapsed);
                for (int i = 1; i < counts.Length; i++)
                {
                    counts[i + 1] += counts[i];
                }

                for (int i = counts.Length - 1; i > 0; i--)
                {
                    counts[i] = counts[i] - 1;
                }

                counts[0] = 0;

                var Trie = new BufferTrie <uint>(20, (array, offset, length) =>
                {
                    for (int i = offset; i < offset + length; i++)
                    {
                        hilbertTransformed[counts[array[i].Index]++] = array[i].Value;
                    }
                });

                for (int i = counts.Length - 1; i > 0; i--)
                {
                    counts[i] = counts[i] - 1;
                }

                counts[0] = 0;

                var buffer = new BufferTrie <uint> .Pair[5000000];

                SingleThreaded.ScanGraph(twitterFile, (vertex, degree, offset, neighbors) =>
                {
                    for (int i = 0; i < degree; i++)
                    {
                        var result = HilbertCurve.xy2dByte(names[vertex], names[neighbors[offset + i]]);

                        buffer[i] = new BufferTrie <uint> .Pair((int)(result >> 32), (uint)(result & 0xFFFF));
                    }

                    Trie.Insert(buffer, 0, degree);
                });

                Trie.Flush();

                Console.WriteLine("{0}\tEdges partitioned", stopwatch.Elapsed);

                using (var upper = new System.IO.BinaryWriter(System.IO.File.OpenWrite("twitter-hilbert-upper")))
                {
                    for (uint i = 0; i < counts.Length - 1; i++)
                    {
                        if (counts[i] < counts[i + 1])
                        {
                            uint x = 0, y = 0;
                            HilbertCurve.d2xyByte((i << 32), out x, out y);

                            upper.Write(x);
                            upper.Write(y);
                            upper.Write(counts[i + 1] - counts[i]);
                        }
                    }
                }

                using (var lower = new System.IO.BinaryWriter(System.IO.File.OpenWrite("twitter-hilbert-lower")))
                {
                    for (uint i = 0; i < counts.Length - 1; i++)
                    {
                        Array.Sort(hilbertTransformed, (int)counts[i], (int)(counts[i + 1] - counts[i]));

                        for (uint j = counts[i]; j < counts[i + 1]; j++)
                        {
                            uint x = 0, y = 0;
                            HilbertCurve.d2xyByte((i << 32) + hilbertTransformed[j], out x, out y);

                            lower.Write((UInt16)(x & 0xFFFF));
                            lower.Write((UInt16)(y & 0xFFFF));
                        }
                    }
                }
            }
            #endregion

            #region hilbert pagerank
            if (algorithm == "hilbertpagerank" && dataset == "livejournal")
            {
                unsafe
                {
                    var stopwatch = System.Diagnostics.Stopwatch.StartNew();

                    var nodes = (uint)42000000;

#if UseLargePages
                    var srcRanks = LargePages.AllocateFloats(nodes);
                    var dstRanks = LargePages.AllocateFloats(nodes);
#else
                    var srcRanks = new float[nodes];
                    var dstRanks = new float[nodes];
#endif
                    for (int i = 0; i < nodes; i++)
                    {
                        srcRanks[i] = 1.0f;
                    }

                    SingleThreaded.HilbertPagerank(@"livejournal-hilbert", dstRanks, srcRanks, nodes, 0.85f);

                    Console.WriteLine(stopwatch.Elapsed);
                }
            }

            if (algorithm == "hilbertpagerank" && dataset == "twitter")
            {
                unsafe
                {
                    var stopwatch = System.Diagnostics.Stopwatch.StartNew();

                    var nodes = (uint)42000000;

#if UseLargePages
                    var srcRanks = LargePages.AllocateFloats(nodes);
                    var dstRanks = LargePages.AllocateFloats(nodes);
#else
                    var srcRanks = new float[nodes];
                    var dstRanks = new float[nodes];
#endif
                    for (int i = 0; i < nodes; i++)
                    {
                        srcRanks[i] = 1.0f;
                    }

                    SingleThreaded.HilbertPagerank(@"twitter-hilbert", dstRanks, srcRanks, nodes, 0.85f);

                    Console.WriteLine(stopwatch.Elapsed);
                }
            }

            if (algorithm == "hilbertpagerank" && dataset == "uk-2007-05")
            {
                unsafe
                {
                    var stopwatch = System.Diagnostics.Stopwatch.StartNew();

                    var nodes = (uint)106000000;

                    var srcRanks = LargePages.AllocateFloats(nodes);
                    var dstRanks = LargePages.AllocateFloats(nodes);

                    for (int i = 0; i < nodes; i++)
                    {
                        srcRanks[i] = 1.0f;
                    }

                    SingleThreaded.MultiHilbertPagerank(@"uk-2007-05-hilbert", dstRanks, srcRanks, nodes, 0.85f);

                    Console.WriteLine(stopwatch.Elapsed);
                }
            }
            #endregion

            #region hilbert union find
            if (algorithm == "hilbertunionfind" && dataset == "twitter")
            {
                unsafe
                {
                    var nodes = (uint)42000000;

                    var stopwatch = System.Diagnostics.Stopwatch.StartNew();
                    SingleThreaded.HilbertUnionFind2(@"twitter-hilbert", nodes);
                    Console.WriteLine(stopwatch.Elapsed);
                }
            }

            if (algorithm == "hilbertunionfind" && dataset == "uk-2007-05")
            {
                unsafe
                {
                    var nodes = (uint)106000000;

                    var stopwatch = System.Diagnostics.Stopwatch.StartNew();
                    SingleThreaded.MultiHilbertCC(@"uk-2007-05-hilbert", nodes);
                    Console.WriteLine(stopwatch.Elapsed);
                }
            }

            if (algorithm == "hilbertunionfind" && dataset == "livejournal")
            {
                unsafe
                {
                    var nodes = (uint)42000000;

                    var stopwatch = System.Diagnostics.Stopwatch.StartNew();
                    SingleThreaded.HilbertUnionFind(@"livejournal-hilbert-upper", "livejournal-hilbert-lower", nodes);
                    Console.WriteLine(stopwatch.Elapsed);
                }
            }
            #endregion

            #region page rank
            if (algorithm == "pagerank" && dataset == "uk-2007-05")
            {
                unsafe
                {
                    var stopwatch = System.Diagnostics.Stopwatch.StartNew();

                    var nodes = (uint)106000000;

#if UseLargePages
                    var srcRanks = LargePages.AllocateFloats(nodes);
                    var dstRanks = LargePages.AllocateFloats(nodes);
#else
                    var srcRanks = new float[nodes];
                    var dstRanks = new float[nodes];
#endif
                    SingleThreaded.PageRankFromDisk(ukFile, dstRanks, srcRanks, nodes, 0.85f);

                    Console.WriteLine(stopwatch.Elapsed);
                }
            }

            if (algorithm == "pagerank" && dataset == "twitter")
            {
                unsafe
                {
                    var stopwatch = System.Diagnostics.Stopwatch.StartNew();

                    var nodes = (uint)65000000;

#if UseLargePages
                    var srcRanks = LargePages.AllocateFloats(nodes);
                    var dstRanks = LargePages.AllocateFloats(nodes);
#else
                    var srcRanks = new float[nodes];
                    var dstRanks = new float[nodes];
#endif
                    SingleThreaded.PageRankFromDisk(twitterFile, dstRanks, srcRanks, nodes, 0.85f);

                    Console.WriteLine(stopwatch.Elapsed);
                }
            }

            if (algorithm == "pagerank" && dataset == "livejournal")
            {
                var stopwatch = System.Diagnostics.Stopwatch.StartNew();

                unsafe
                {
                    var nodes = (uint)65000000;

#if UseLargePages
                    var srcRanks = LargePages.AllocateFloats(nodes);
                    var dstRanks = LargePages.AllocateFloats(nodes);
#else
                    var srcRanks = new float[nodes];
                    var dstRanks = new float[nodes];
#endif
                    for (int i = 0; i < 20; i++)
                    {
                        SingleThreaded.PageRankStep(livejournalFile, dstRanks, srcRanks, nodes, 0.85f);
                        Console.WriteLine("{0}\tIteration {1}", stopwatch.Elapsed, i);
                    }
                }

                Console.WriteLine(stopwatch.Elapsed);
            }
            #endregion

            #region connected components
            if (algorithm == "connectedcomponents" && dataset == "twitter")
            {
                var stopwatch = System.Diagnostics.Stopwatch.StartNew();
                SingleThreaded.ConnectedComponents(twitterFile, 65000000);
                Console.WriteLine(stopwatch.Elapsed);
            }

            if (algorithm == "connectedcomponents" && dataset == "uk-2007-05")
            {
                var stopwatch = System.Diagnostics.Stopwatch.StartNew();
                SingleThreaded.ClumsyCC(ukFile, 106000000);
                Console.WriteLine(stopwatch.Elapsed);
            }

            if (algorithm == "connectedcomponents" && dataset == "livejournal")
            {
                var stopwatch = System.Diagnostics.Stopwatch.StartNew();
                SingleThreaded.ConnectedComponents(livejournalFile, 6500000);
                Console.WriteLine(stopwatch.Elapsed);
            }
            #endregion

            #region maximal independent set
            if (algorithm == "maximalindependentset" && dataset == "twitter")
            {
                var stopwatch = System.Diagnostics.Stopwatch.StartNew();
                SingleThreaded.MaximalIndependentSet(twitterFile, 65000000);
                Console.WriteLine(stopwatch.Elapsed);
            }

            if (algorithm == "maximalindependentset" && dataset == "livejournal")
            {
                var stopwatch = System.Diagnostics.Stopwatch.StartNew();
                SingleThreaded.MaximalIndependentSet(livejournalFile, 6500000);
                Console.WriteLine(stopwatch.Elapsed);
            }
            #endregion
        }