public unsafe static void PageRankFromDisk(string filename, float *a, float *b, uint nodes, float reset)
        {
            var stopwatch = System.Diagnostics.Stopwatch.StartNew();

            for (int i = 0; i < nodes; i++)
            {
                a[i] = reset;
            }

            var graph = new FileGraphScanner(filename);

            for (int iteration = 0; iteration < 20; iteration++)
            {
                var startTime = stopwatch.Elapsed;

                graph.ForEach((vertex, degree) =>
                {
                    b[vertex] = (1.0f - reset) * a[vertex] / degree;
                    a[vertex] = reset;
                });

                graph.ForEach((vertex, degree, offset, neighbors) =>
                {
                    for (int i = 0; i < degree; i++)
                    {
                        a[neighbors[offset + i]] += b[vertex];
                    }
                });

                Console.WriteLine("{0}\tIteration {1}", stopwatch.Elapsed - startTime, iteration);
            }

            Console.WriteLine(stopwatch.Elapsed);
        }
示例#2
0
        public static void TestTransformed(string prefix)
        {
            var stopwatch = System.Diagnostics.Stopwatch.StartNew();

            var high32MetaData = UnbufferedIO.ReadFile <uint>(prefix + "-high32");

            var next16Reader = new UnbufferedIO.SequentialReader(prefix + "-next16");
            var next08Reader = new UnbufferedIO.SequentialReader(prefix + "-next08");
            var last08Reader = new UnbufferedIO.SequentialReader(prefix + "-last08");

            var next16Buffer = new ushort[] { };
            var next08Buffer = new byte[] { };
            var last08Buffer = new byte[] { };

            var iDegrees = new uint[106000000];
            var oDegrees = new uint[106000000];

            var edgeCount = 0L;

            for (int i = 0; i < high32MetaData.Length; i += 2)
            {
                var high32value = high32MetaData[i];
                var high32count = (int)high32MetaData[i + 1];

                if (next16Buffer.Length < 2 * high32count)
                {
                    next16Buffer = new ushort[2 * high32count];
                }

                var readNext16 = next16Reader.Read <ushort>(next16Buffer, 0, 2 * high32count);
                if (readNext16 != 2 * high32count)
                {
                    Console.WriteLine("Read error, next16; requested {0}, read {1}", 2 * high32count, readNext16);
                }


                var next16count = 0;
                for (int j = 0; j < high32count; j++)
                {
                    next16count += (int)(next16Buffer[2 * j + 1] == 0 ? ((uint)ushort.MaxValue) + 1 : next16Buffer[2 * j + 1]);
                }

                if (next08Buffer.Length < 2 * next16count)
                {
                    next08Buffer = new byte[2 * next16count];
                }

                var readNext08 = next08Reader.Read <byte>(next08Buffer, 0, 2 * next16count);
                if (readNext08 != 2 * next16count)
                {
                    Console.WriteLine("Read error, next08; requested {0}, read {1}", 2 * next16count, readNext08);
                }


                var next08count = 0;
                for (int k = 0; k < next16count; k++)
                {
                    next08count += (int)(next08Buffer[2 * k + 1] == 0 ? ((uint)byte.MaxValue) + 1 : next08Buffer[2 * k + 1]);
                }

                if (last08Buffer.Length < next08count)
                {
                    last08Buffer = new byte[next08count];
                }

                var readLast08 = last08Reader.Read <byte>(last08Buffer, 0, next08count);
                if (readLast08 != next08count)
                {
                    Console.WriteLine("Read error, last08; requested {0}, read {1}", next08count, readLast08);
                }


                var next16Offset = 0;
                var next08Offset = 0;
                var last08Offset = 0;

                // do work for all these dudes
                var xHigh32 = (high32value & 0xFFFF0000) << 0;
                var yHigh32 = (high32value & 0x0000FFFF) << 16;

                for (int j = 0; j < high32count; j++)
                {
                    var xNext16 = xHigh32 + ((next16Buffer[next16Offset + 2 * j] & 0xFF00) << 0);
                    var yNext16 = yHigh32 + ((next16Buffer[next16Offset + 2 * j] & 0x00FF) << 8);

                    next16count = next16Buffer[next16Offset + 2 * j + 1];
                    if (next16count == 0)
                    {
                        next16count = ((int)ushort.MaxValue) + 1;
                    }

                    for (int k = 0; k < next16count; k++)
                    {
                        var xNext08 = xNext16 + ((next08Buffer[next08Offset + 2 * k] & 0xF0) << 0);
                        var yNext08 = yNext16 + ((next08Buffer[next08Offset + 2 * k] & 0x0F) << 4);

                        next08count = next08Buffer[next08Offset + 2 * k + 1];
                        if (next08count == 0)
                        {
                            next08count = ((int)byte.MaxValue) + 1;
                        }

                        for (int l = 0; l < next08count; l++)
                        {
                            var x = xNext08 + ((last08Buffer[last08Offset + l] & 0xF0) >> 4);
                            var y = yNext08 + ((last08Buffer[last08Offset + l] & 0x0F) >> 0);

                            iDegrees[y]++;
                            oDegrees[x]++;

                            edgeCount++;
                        }

                        last08Offset += next08count;
                    }

                    next08Offset += 2 * next16count;
                }
            }

            Console.WriteLine("{1}\tProcessed\t{0} edges", edgeCount, stopwatch.Elapsed);

            next16Reader.Dispose();
            next08Reader.Dispose();
            last08Reader.Dispose();


            var graph = new FileGraphScanner(prefix);

            graph.ForEach((vertex, degree, offset, neighbors) =>
            {
                for (int i = 0; i < degree; i++)
                {
                    iDegrees[neighbors[offset + i]]--;
                    oDegrees[vertex]--;
                }
            });

            var iErrors = iDegrees.Count(x => x > 0);
            var oErrors = oDegrees.Count(x => x > 0);

            Console.WriteLine("iErrors: {0}\toErrors: {1}", iErrors, oErrors);

            Console.WriteLine("{0}\tCounted {1} edges", stopwatch.Elapsed, edgeCount);
        }
示例#3
0
        public static void ConvertToHilbert(this string prefix)
        {
            var graph = new FileGraphScanner(prefix);

            var High32 = new Dictionary <UInt32, UInt32>();

            graph.ForEach((vertex, degree, offset, neighbors) =>
            {
                //if (vertex < 1000000)
                {
                    for (int i = 0; i < degree; i++)
                    {
                        var neighbor = neighbors[offset + i];
                        var high32   = (uint)(HilbertCurve.xy2dByte((uint)vertex, (uint)neighbor) >> 32);

                        if (!High32.ContainsKey(high32))
                        {
                            High32.Add(high32, 0);
                        }

                        High32[high32] = High32[high32] + 1;
                    }
                }
            });

            Console.WriteLine("Assesed prefix sizes");

            var pairs = High32.OrderBy(x => x.Key).ToArray();

            var edgeCount = 0L;

            var lowIndex = (uint)0;

            var high32Stream = new System.IO.BinaryWriter(System.IO.File.OpenWrite(prefix + "-high32"));
            var next16Stream = new System.IO.BinaryWriter(System.IO.File.OpenWrite(prefix + "-next16"));
            var next08Stream = new System.IO.BinaryWriter(System.IO.File.OpenWrite(prefix + "-next08"));
            var last08Stream = new System.IO.BinaryWriter(System.IO.File.OpenWrite(prefix + "-last08"));

            for (uint i = 0; i < pairs.Length; i++)
            {
                // if we would have too many edges, do work
                if (edgeCount + pairs[i].Value > 1 << 29)
                {
                    var edges  = new ulong[edgeCount];
                    var cursor = 0;

                    graph.ForEach((vertex, degree, offset, neighbors) =>
                    {
                        //if (vertex < 1000000)
                        {
                            for (int j = 0; j < degree; j++)
                            {
                                var transform = HilbertCurve.xy2dByte((uint)vertex, (uint)neighbors[offset + j]);
                                if ((transform >> 32) >= pairs[lowIndex].Key && (transform >> 32) < pairs[i].Key)
                                {
                                    edges[cursor++] = transform;
                                }
                            }
                        }
                    });

                    if (cursor != edges.Length)
                    {
                        Console.WriteLine("Somehow read the wrong number of edges {0} vs {1}", cursor, edges.Length);
                    }

                    Array.Sort(edges);

                    Console.WriteLine("About to process {0} high32 blocks", i - lowIndex);

                    // traverse edges in order and write out when interesting things happen.
                    WriteTransformed(edges, high32Stream, next16Stream, next08Stream, last08Stream);

                    edgeCount = 0;
                    lowIndex  = i;
                }

                edgeCount += pairs[i].Value;
            }

            var edges2  = new ulong[edgeCount];
            var cursor2 = 0;

            graph.ForEach((vertex, degree, offset, neighbors) =>
            {
                //if (vertex < 1000000)
                {
                    for (int j = 0; j < degree; j++)
                    {
                        var transform = HilbertCurve.xy2dByte((uint)vertex, (uint)neighbors[offset + j]);

                        if ((transform >> 32) >= pairs[lowIndex].Key)
                        {
                            edges2[cursor2++] = transform;
                        }
                    }
                }
            });

            Array.Sort(edges2);

            if (cursor2 != edges2.Length)
            {
                Console.WriteLine("Somehow read the wrong number of edges {0} vs {1}", cursor2, edges2.Length);
            }


            // traverse edges in order and write out when interesting things happen.
            WriteTransformed(edges2, high32Stream, next16Stream, next08Stream, last08Stream);

            high32Stream.Close();
            next16Stream.Close();
            next08Stream.Close();
            last08Stream.Close();
        }