コード例 #1
0
        public void ForEach(Action <int, int, int, int[]> perVertexLogic)
        {
            var metadata = UnbufferedIO.ReadFile <int>(this.filePrefix + "-nodes");

            var maxDegree = 0;

            for (int i = 0; i < metadata.Length; i += 2)
            {
                maxDegree = Math.Max(maxDegree, metadata[i + 1]);
            }

            var neighbors       = new int[2 * maxDegree];
            var neighborsOffset = 0;

            using (var reader = new UnbufferedIO.SequentialReader(this.filePrefix + "-edges"))
            {
                reader.Read(neighbors, 0, neighbors.Length);

                for (int i = 0; i < metadata.Length; i += 2)
                {
                    var vertex = metadata[i + 0];
                    var degree = metadata[i + 1];

                    if (neighborsOffset + degree > neighbors.Length - 1)
                    {
                        Array.Copy(neighbors, neighborsOffset, neighbors, 0, neighbors.Length - neighborsOffset);
                        reader.Read(neighbors, neighbors.Length - neighborsOffset, neighborsOffset);
                        neighborsOffset = 0;
                    }

                    perVertexLogic(vertex, degree, neighborsOffset, neighbors);
                    neighborsOffset += degree;
                }
            }
        }
コード例 #2
0
        public void ForEach(Action <int, int> perVertexLogic)
        {
            var metadata = UnbufferedIO.ReadFile <int>(this.filePrefix + "-nodes");

            for (int i = 0; i < metadata.Length; i += 2)
            {
                perVertexLogic(metadata[i], metadata[i + 1]);
            }
        }
コード例 #3
0
        public static void ScanGraph(string filename, Action <int, int> perVertexLogic)
        {
            var metadata = UnbufferedIO.ReadFile <int>(filename + "-nodes");

            for (int i = 0; i < metadata.Length; i += 2)
            {
                perVertexLogic(metadata[i], metadata[i + 1]);
            }
        }
コード例 #4
0
        public static void TestTransformedAlt(string prefix)
        {
            var stopwatch = System.Diagnostics.Stopwatch.StartNew();

            var high32MetaData = UnbufferedIO.ReadFile <uint>(prefix + "-high32");

            var high32Counts = 0L;

            for (int i = 0; i < high32MetaData.Length; i += 2)
            {
                high32Counts += high32MetaData[i + 1];
            }

            Console.WriteLine("Read {0} high32, expecting {1} next16", high32MetaData.Length / 2, high32Counts);

            var next16Total  = 0L;
            var next16Count  = 0L;
            var next16Buffer = new ushort[1 << 20];

            using (var next16Reader = new UnbufferedIO.SequentialReader(prefix + "-next16"))
            {
                for (var read = next16Reader.Read <ushort>(next16Buffer, 0, 1 << 20); read > 0; read = next16Reader.Read <ushort>(next16Buffer, 0, 1 << 20))
                {
                    next16Total += read / 2;
                    for (int i = 0; i < read; i += 2)
                    {
                        next16Count += next16Buffer[i + 1];
                    }
                }
            }

            Console.WriteLine("Read {0} next16, expecting {1} next08", next16Total, next16Count);

            var next08Total  = 0L;
            var next08Count  = 0L;
            var next08Buffer = new byte[1 << 20];

            using (var next08Reader = new UnbufferedIO.SequentialReader(prefix + "-next08"))
            {
                for (var read = next08Reader.Read <byte>(next08Buffer, 0, 1 << 20); read > 0; read = next08Reader.Read <byte>(next08Buffer, 0, 1 << 20))
                {
                    next08Total += read / 2;
                    for (int i = 0; i < read; i += 2)
                    {
                        next08Count += next08Buffer[i + 1] == 0 ? 256 : (int)next08Buffer[i + 1];
                    }
                }
            }

            Console.WriteLine("Read {0} next08, expecting {1} last08", next08Total, next08Count);


            Console.WriteLine(stopwatch.Elapsed);
        }
コード例 #5
0
        public static unsafe void HilbertPagerank(string filename, float *a, float *b, uint nodes, float reset)
        {
            var stopwatch = System.Diagnostics.Stopwatch.StartNew();

            var degrees = LargePages.AllocateInts(nodes);
            //var degrees = new int[nodes];

            var metadata = UnbufferedIO.ReadFile <uint>(filename + "-upper");
            var edges    = UnbufferedIO.ReadFile <uint>(filename + "-lower");

            Console.WriteLine("{2}\tRead header of {0} blocks, for {1} edges", metadata.Length / 3, edges.Length, stopwatch.Elapsed);

            ScanGraphHilbert(metadata, edges, (srcUpper, tgtUpper, offset, count, edgeArray) =>
            {
                for (var j = offset; j < offset + count; j++)
                {
                    degrees[srcUpper + (edges[j] & 0xFFFF)]++;
                }
            });

            Console.WriteLine("{0}\tDegrees calculated", stopwatch.Elapsed);

            for (int iteration = 0; iteration < 20; iteration++)
            {
                var startTime = stopwatch.Elapsed;

                for (int i = 0; i < nodes; i++)
                {
                    b[i] = (1.0f - reset) * a[i] / degrees[i];
                    a[i] = reset;
                }

                ScanGraphHilbert(metadata, edges, (sourceUpper, targetUpper, offset, count, edgeArray) =>
                {
                    for (var j = offset; j < offset + count; j++)
                    {
                        a[targetUpper + (edges[j] >> 16)] += b[sourceUpper + (edges[j] & 0xFFFF)];
                    }
                });

                Console.WriteLine("{0}\tIteration {1} in {2}", stopwatch.Elapsed, iteration, stopwatch.Elapsed - startTime);
            }
        }
コード例 #6
0
        public unsafe static void PageRankStep(string filename, float *a, float *b, uint nodes, float reset)
        {
            var stopwatch = System.Diagnostics.Stopwatch.StartNew();

            var nodeArray = UnbufferedIO.ReadFile <int>(filename + "-nodes");
            var edgeArray = UnbufferedIO.ReadFile <int>(filename + "-edges");

            for (int i = 0; i < nodes; i++)
            {
                a[i] = reset;
            }

            for (int iteration = 0; iteration < 20; iteration++)
            {
                var startTime = stopwatch.Elapsed;

                for (int i = 0; i < nodes; i++)
                {
                    b[i] = a[i];
                    a[i] = 0.0f;
                }

                // apply per-vertex pagerank logic across the graph
                ScanGraph(nodeArray, edgeArray, (vertex, degree, offset, neighbors) =>
                {
                    if (b[vertex] > 0.0f)
                    {
                        var update = (1.0f - reset) * b[vertex] / degree;
                        for (int i = 0; i < degree; i++)
                        {
                            a[neighbors[offset + i]] += update;
                        }
                    }
                });

                Console.WriteLine("{0}\tIteration {1}", stopwatch.Elapsed - startTime, iteration);
            }

            Console.WriteLine(stopwatch.Elapsed);
        }
コード例 #7
0
        public MultiHilbertScanner(string prefix)
        {
            this.prefix = prefix;

            this.high32Buffer = UnbufferedIO.ReadFile <uint>(prefix + "-high32");
            this.next16Buffer = UnbufferedIO.ReadFile <ushort>(prefix + "-next16");
            this.next08Buffer = UnbufferedIO.ReadFile <byte>(prefix + "-next08");

            this.last08ToRead = new int[high32Buffer.Length / 2];

            var next16Offset = 0;
            var next08Offset = 0;

            for (int i = 0; i < high32Buffer.Length; i += 2)
            {
                var high32value = high32Buffer[i];
                var high32count = (int)high32Buffer[i + 1];

                var next16count = 0;
                for (int j = 0; j < high32count; j++)
                {
                    next16count += (int)(next16Buffer[next16Offset + 2 * j + 1] == 0 ? ((uint)ushort.MaxValue) + 1 : next16Buffer[next16Offset + 2 * j + 1]);
                }

                next16Offset += 2 * high32count;

                var next08count = 0;
                for (int k = 0; k < next16count; k++)
                {
                    next08count += (int)(next08Buffer[next08Offset + 2 * k + 1] == 0 ? ((uint)byte.MaxValue) + 1 : next08Buffer[next08Offset + 2 * k + 1]);
                }

                next08Offset += 2 * next16count;

                last08ToRead[i / 2] = next08count;
            }

            this.last08Buffer = new byte[last08ToRead.Max()];
        }
コード例 #8
0
        public static void ScanGraphHilbert(string filename, Action <uint, uint, uint, uint, uint[]> perVertexLogic)
        {
            var metadata = UnbufferedIO.ReadFile <uint>(filename + "-upper");

            var maxCount = (uint)0;

            for (int i = 0; i < metadata.Length; i += 3)
            {
                maxCount = Math.Max(maxCount, metadata[i + 2]);
            }

            var edges       = new uint[2 * maxCount];
            var edgesOffset = (uint)0;

            using (var reader = new UnbufferedIO.SequentialReader(filename + "-lower"))
            {
                reader.Read(edges, 0, edges.Length);

                for (int i = 0; i < metadata.Length; i += 3)
                {
                    var sourceUpper = metadata[i + 0] << 16;
                    var targetUpper = metadata[i + 1] << 16;

                    var count = metadata[i + 2];

                    if (edgesOffset + count > edges.Length - 1)
                    {
                        Array.Copy(edges, edgesOffset, edges, 0, edges.Length - edgesOffset);
                        reader.Read(edges, (int)(edges.Length - edgesOffset), (int)edgesOffset);
                        edgesOffset = 0;
                    }

                    perVertexLogic(sourceUpper, targetUpper, edgesOffset, count, edges);
                    edgesOffset += count;
                }
            }
        }
コード例 #9
0
        public static void TestTransformed(string prefix)
        {
            var stopwatch = System.Diagnostics.Stopwatch.StartNew();

            var high32MetaData = UnbufferedIO.ReadFile <uint>(prefix + "-high32");

            var next16Reader = new UnbufferedIO.SequentialReader(prefix + "-next16");
            var next08Reader = new UnbufferedIO.SequentialReader(prefix + "-next08");
            var last08Reader = new UnbufferedIO.SequentialReader(prefix + "-last08");

            var next16Buffer = new ushort[] { };
            var next08Buffer = new byte[] { };
            var last08Buffer = new byte[] { };

            var iDegrees = new uint[106000000];
            var oDegrees = new uint[106000000];

            var edgeCount = 0L;

            for (int i = 0; i < high32MetaData.Length; i += 2)
            {
                var high32value = high32MetaData[i];
                var high32count = (int)high32MetaData[i + 1];

                if (next16Buffer.Length < 2 * high32count)
                {
                    next16Buffer = new ushort[2 * high32count];
                }

                var readNext16 = next16Reader.Read <ushort>(next16Buffer, 0, 2 * high32count);
                if (readNext16 != 2 * high32count)
                {
                    Console.WriteLine("Read error, next16; requested {0}, read {1}", 2 * high32count, readNext16);
                }


                var next16count = 0;
                for (int j = 0; j < high32count; j++)
                {
                    next16count += (int)(next16Buffer[2 * j + 1] == 0 ? ((uint)ushort.MaxValue) + 1 : next16Buffer[2 * j + 1]);
                }

                if (next08Buffer.Length < 2 * next16count)
                {
                    next08Buffer = new byte[2 * next16count];
                }

                var readNext08 = next08Reader.Read <byte>(next08Buffer, 0, 2 * next16count);
                if (readNext08 != 2 * next16count)
                {
                    Console.WriteLine("Read error, next08; requested {0}, read {1}", 2 * next16count, readNext08);
                }


                var next08count = 0;
                for (int k = 0; k < next16count; k++)
                {
                    next08count += (int)(next08Buffer[2 * k + 1] == 0 ? ((uint)byte.MaxValue) + 1 : next08Buffer[2 * k + 1]);
                }

                if (last08Buffer.Length < next08count)
                {
                    last08Buffer = new byte[next08count];
                }

                var readLast08 = last08Reader.Read <byte>(last08Buffer, 0, next08count);
                if (readLast08 != next08count)
                {
                    Console.WriteLine("Read error, last08; requested {0}, read {1}", next08count, readLast08);
                }


                var next16Offset = 0;
                var next08Offset = 0;
                var last08Offset = 0;

                // do work for all these dudes
                var xHigh32 = (high32value & 0xFFFF0000) << 0;
                var yHigh32 = (high32value & 0x0000FFFF) << 16;

                for (int j = 0; j < high32count; j++)
                {
                    var xNext16 = xHigh32 + ((next16Buffer[next16Offset + 2 * j] & 0xFF00) << 0);
                    var yNext16 = yHigh32 + ((next16Buffer[next16Offset + 2 * j] & 0x00FF) << 8);

                    next16count = next16Buffer[next16Offset + 2 * j + 1];
                    if (next16count == 0)
                    {
                        next16count = ((int)ushort.MaxValue) + 1;
                    }

                    for (int k = 0; k < next16count; k++)
                    {
                        var xNext08 = xNext16 + ((next08Buffer[next08Offset + 2 * k] & 0xF0) << 0);
                        var yNext08 = yNext16 + ((next08Buffer[next08Offset + 2 * k] & 0x0F) << 4);

                        next08count = next08Buffer[next08Offset + 2 * k + 1];
                        if (next08count == 0)
                        {
                            next08count = ((int)byte.MaxValue) + 1;
                        }

                        for (int l = 0; l < next08count; l++)
                        {
                            var x = xNext08 + ((last08Buffer[last08Offset + l] & 0xF0) >> 4);
                            var y = yNext08 + ((last08Buffer[last08Offset + l] & 0x0F) >> 0);

                            iDegrees[y]++;
                            oDegrees[x]++;

                            edgeCount++;
                        }

                        last08Offset += next08count;
                    }

                    next08Offset += 2 * next16count;
                }
            }

            Console.WriteLine("{1}\tProcessed\t{0} edges", edgeCount, stopwatch.Elapsed);

            next16Reader.Dispose();
            next08Reader.Dispose();
            last08Reader.Dispose();


            var graph = new FileGraphScanner(prefix);

            graph.ForEach((vertex, degree, offset, neighbors) =>
            {
                for (int i = 0; i < degree; i++)
                {
                    iDegrees[neighbors[offset + i]]--;
                    oDegrees[vertex]--;
                }
            });

            var iErrors = iDegrees.Count(x => x > 0);
            var oErrors = oDegrees.Count(x => x > 0);

            Console.WriteLine("iErrors: {0}\toErrors: {1}", iErrors, oErrors);

            Console.WriteLine("{0}\tCounted {1} edges", stopwatch.Elapsed, edgeCount);
        }
コード例 #10
0
        public static void ScanAlt(this string prefix, Action <uint, uint, int, int, byte[]> perEdgeAction)
        {
            var high32Buffer = UnbufferedIO.ReadFile <uint>(prefix + "-high32");
            var next16Buffer = UnbufferedIO.ReadFile <ushort>(prefix + "-next16");
            var next08Buffer = UnbufferedIO.ReadFile <byte>(prefix + "-next08");

            var last08ToRead = new int[high32Buffer.Length / 2];

            var next16Offset = 0;
            var next08Offset = 0;

            for (int i = 0; i < high32Buffer.Length; i += 2)
            {
                var high32value = high32Buffer[i];
                var high32count = (int)high32Buffer[i + 1];

                var next16count = 0;
                for (int j = 0; j < high32count; j++)
                {
                    next16count += (int)(next16Buffer[next16Offset + 2 * j + 1] == 0 ? ((uint)ushort.MaxValue) + 1 : next16Buffer[next16Offset + 2 * j + 1]);
                }

                next16Offset += 2 * high32count;

                var next08count = 0;
                for (int k = 0; k < next16count; k++)
                {
                    next08count += (int)(next08Buffer[next08Offset + 2 * k + 1] == 0 ? ((uint)byte.MaxValue) + 1 : next08Buffer[next08Offset + 2 * k + 1]);
                }

                next08Offset += 2 * next16count;

                last08ToRead[i / 2] = next08count;
            }

            next16Offset = 0;
            next08Offset = 0;

            var last08Reader = new UnbufferedIO.SequentialReader(prefix + "-last08");
            var last08Buffer = new byte[last08ToRead.Max()];

            for (int i = 0; i < high32Buffer.Length; i += 2)
            {
                var high32value = high32Buffer[i];
                var high32count = (int)high32Buffer[i + 1];

                var readLast08 = last08Reader.Read <byte>(last08Buffer, 0, last08ToRead[i / 2]);
                if (readLast08 != last08ToRead[i / 2])
                {
                    Console.WriteLine("Read error, last08; requested {0}, read {1}", last08ToRead[i / 2], readLast08);
                }

                var last08Offset = 0;

                // do work for all these dudes
                var xHigh32 = (high32value & 0xFFFF0000) << 0;
                var yHigh32 = (high32value & 0x0000FFFF) << 16;

                for (int j = 0; j < high32count; j++)
                {
                    var xNext16 = xHigh32 + (uint)((next16Buffer[next16Offset + 2 * j] & 0xFF00) << 0);
                    var yNext16 = yHigh32 + (uint)((next16Buffer[next16Offset + 2 * j] & 0x00FF) << 8);

                    var next16count = (int)next16Buffer[next16Offset + 2 * j + 1];
                    if (next16count == 0)
                    {
                        next16count = ((int)ushort.MaxValue) + 1;
                    }

                    for (int k = 0; k < next16count; k++)
                    {
                        var xNext08 = xNext16 + (uint)((next08Buffer[next08Offset + 2 * k] & 0xF0) << 0);
                        var yNext08 = yNext16 + (uint)((next08Buffer[next08Offset + 2 * k] & 0x0F) << 4);

                        var next08count = (int)next08Buffer[next08Offset + 2 * k + 1];
                        if (next08count == 0)
                        {
                            next08count = ((int)byte.MaxValue) + 1;
                        }

                        perEdgeAction(xNext08, yNext08, next08count, last08Offset, last08Buffer);

                        last08Offset += next08count;
                    }

                    next08Offset += 2 * next16count;
                }

                next16Offset += 2 * high32count;
            }

            last08Reader.Dispose();
        }
コード例 #11
0
 public MemoryGraphScanner(string prefix)
 {
     this.metadata  = UnbufferedIO.ReadFile <int>(prefix + "-nodes");
     this.neighbors = UnbufferedIO.ReadFile <int>(prefix + "-edges");
 }