public void ForEach(Action <int, int, int, int[]> perVertexLogic)
{
var metadata = UnbufferedIO.ReadFile <int>(this.filePrefix + "-nodes");
var maxDegree = 0;
for (int i = 0; i < metadata.Length; i += 2)
{
maxDegree = Math.Max(maxDegree, metadata[i + 1]);
}
var neighbors = new int[2 * maxDegree];
var neighborsOffset = 0;
using (var reader = new UnbufferedIO.SequentialReader(this.filePrefix + "-edges"))
{
reader.Read(neighbors, 0, neighbors.Length);
for (int i = 0; i < metadata.Length; i += 2)
{
var vertex = metadata[i + 0];
var degree = metadata[i + 1];
if (neighborsOffset + degree > neighbors.Length - 1)
{
Array.Copy(neighbors, neighborsOffset, neighbors, 0, neighbors.Length - neighborsOffset);
reader.Read(neighbors, neighbors.Length - neighborsOffset, neighborsOffset);
neighborsOffset = 0;
}
perVertexLogic(vertex, degree, neighborsOffset, neighbors);
neighborsOffset += degree;
}
}
}
public static T[] ReadFile <T>(string filename)
{
using (var reader = new UnbufferedIO.SequentialReader(filename))
{
var result = new T[reader.Length / System.Runtime.InteropServices.Marshal.SizeOf(typeof(T))];
// if the thing to read is bigger than 2GB, Buffer.BlockCopy will choke (yay 32 bits)
if (((Int64)result.Length) * System.Runtime.InteropServices.Marshal.SizeOf(typeof(T)) > Int32.MaxValue)
{
var intermediate = new T[1 << 20];
var cursor = 0;
while (reader.Read(intermediate, 0, intermediate.Length) > 0)
{
Array.Copy(intermediate, 0, result, cursor, Math.Min(intermediate.Length, result.Length - cursor));
cursor += intermediate.Length;
}
}
else
{
reader.Read(result, 0, result.Length);
}
return(result);
}
}
public void Scan(Action <uint, uint, int, int, byte[]> perEdgeAction)
{
var next16Offset = 0;
var next08Offset = 0;
using (var last08Reader = new UnbufferedIO.SequentialReader(prefix + "-last08"))
{
for (int i = 0; i < high32Buffer.Length; i += 2)
{
var high32value = high32Buffer[i];
var high32count = (int)high32Buffer[i + 1];
var readLast08 = last08Reader.Read <byte>(last08Buffer, 0, last08ToRead[i / 2]);
if (readLast08 != last08ToRead[i / 2])
{
Console.WriteLine("Read error, last08; requested {0}, read {1}", last08ToRead[i / 2], readLast08);
}
var last08Offset = 0;
// do work for all these dudes
var xHigh32 = (high32value & 0xFFFF0000) << 0;
var yHigh32 = (high32value & 0x0000FFFF) << 16;
for (int j = 0; j < high32count; j++)
{
var xNext16 = xHigh32 + (uint)((next16Buffer[next16Offset + 2 * j] & 0xFF00) << 0);
var yNext16 = yHigh32 + (uint)((next16Buffer[next16Offset + 2 * j] & 0x00FF) << 8);
var next16count = (int)next16Buffer[next16Offset + 2 * j + 1];
if (next16count == 0)
{
next16count = ((int)ushort.MaxValue) + 1;
}
for (int k = 0; k < next16count; k++)
{
var xNext08 = xNext16 + (uint)((next08Buffer[next08Offset + 2 * k] & 0xF0) << 0);
var yNext08 = yNext16 + (uint)((next08Buffer[next08Offset + 2 * k] & 0x0F) << 4);
var next08count = (int)next08Buffer[next08Offset + 2 * k + 1];
if (next08count == 0)
{
next08count = ((int)byte.MaxValue) + 1;
}
perEdgeAction(xNext08, yNext08, next08count, last08Offset, last08Buffer);
last08Offset += next08count;
}
next08Offset += 2 * next16count;
}
next16Offset += 2 * high32count;
}
}
}
public static void TestTransformedAlt(string prefix)
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var high32MetaData = UnbufferedIO.ReadFile <uint>(prefix + "-high32");
var high32Counts = 0L;
for (int i = 0; i < high32MetaData.Length; i += 2)
{
high32Counts += high32MetaData[i + 1];
}
Console.WriteLine("Read {0} high32, expecting {1} next16", high32MetaData.Length / 2, high32Counts);
var next16Total = 0L;
var next16Count = 0L;
var next16Buffer = new ushort[1 << 20];
using (var next16Reader = new UnbufferedIO.SequentialReader(prefix + "-next16"))
{
for (var read = next16Reader.Read <ushort>(next16Buffer, 0, 1 << 20); read > 0; read = next16Reader.Read <ushort>(next16Buffer, 0, 1 << 20))
{
next16Total += read / 2;
for (int i = 0; i < read; i += 2)
{
next16Count += next16Buffer[i + 1];
}
}
}
Console.WriteLine("Read {0} next16, expecting {1} next08", next16Total, next16Count);
var next08Total = 0L;
var next08Count = 0L;
var next08Buffer = new byte[1 << 20];
using (var next08Reader = new UnbufferedIO.SequentialReader(prefix + "-next08"))
{
for (var read = next08Reader.Read <byte>(next08Buffer, 0, 1 << 20); read > 0; read = next08Reader.Read <byte>(next08Buffer, 0, 1 << 20))
{
next08Total += read / 2;
for (int i = 0; i < read; i += 2)
{
next08Count += next08Buffer[i + 1] == 0 ? 256 : (int)next08Buffer[i + 1];
}
}
}
Console.WriteLine("Read {0} next08, expecting {1} last08", next08Total, next08Count);
Console.WriteLine(stopwatch.Elapsed);
}
public static void ConvertGraph(string filename)
{
using (var nodeWriter = new BinaryWriter(File.OpenWrite(filename + "-nodes")))
{
using (var edgeWriter = new BinaryWriter(File.OpenWrite(filename + "-edges")))
{
using (var reader = new UnbufferedIO.SequentialReader(filename))
{
var ints = new int[2];
var bytesToRead = reader.Length - 8;
// read vertex name and degree
reader.Read(ints, 0, 2);
var vertex = ints[0];
var degree = ints[1];
while (bytesToRead > 0)
{
// allocate space if needed
if (ints.Length < degree + 2)
{
ints = new int[degree + 2];
}
bytesToRead -= 4 * reader.Read(ints, 0, (degree + 2));
nodeWriter.Write(vertex);
nodeWriter.Write(degree);
for (int i = 0; i < degree; i++)
{
edgeWriter.Write(ints[i]);
}
vertex = ints[degree];
degree = ints[degree + 1];
}
}
}
}
}
public static void ScanGraphHilbert(string filename, Action <uint, uint, uint, uint, uint[]> perVertexLogic)
{
var metadata = UnbufferedIO.ReadFile <uint>(filename + "-upper");
var maxCount = (uint)0;
for (int i = 0; i < metadata.Length; i += 3)
{
maxCount = Math.Max(maxCount, metadata[i + 2]);
}
var edges = new uint[2 * maxCount];
var edgesOffset = (uint)0;
using (var reader = new UnbufferedIO.SequentialReader(filename + "-lower"))
{
reader.Read(edges, 0, edges.Length);
for (int i = 0; i < metadata.Length; i += 3)
{
var sourceUpper = metadata[i + 0] << 16;
var targetUpper = metadata[i + 1] << 16;
var count = metadata[i + 2];
if (edgesOffset + count > edges.Length - 1)
{
Array.Copy(edges, edgesOffset, edges, 0, edges.Length - edgesOffset);
reader.Read(edges, (int)(edges.Length - edgesOffset), (int)edgesOffset);
edgesOffset = 0;
}
perVertexLogic(sourceUpper, targetUpper, edgesOffset, count, edges);
edgesOffset += count;
}
}
}
public static void TestTransformed(string prefix)
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var high32MetaData = UnbufferedIO.ReadFile <uint>(prefix + "-high32");
var next16Reader = new UnbufferedIO.SequentialReader(prefix + "-next16");
var next08Reader = new UnbufferedIO.SequentialReader(prefix + "-next08");
var last08Reader = new UnbufferedIO.SequentialReader(prefix + "-last08");
var next16Buffer = new ushort[] { };
var next08Buffer = new byte[] { };
var last08Buffer = new byte[] { };
var iDegrees = new uint[106000000];
var oDegrees = new uint[106000000];
var edgeCount = 0L;
for (int i = 0; i < high32MetaData.Length; i += 2)
{
var high32value = high32MetaData[i];
var high32count = (int)high32MetaData[i + 1];
if (next16Buffer.Length < 2 * high32count)
{
next16Buffer = new ushort[2 * high32count];
}
var readNext16 = next16Reader.Read <ushort>(next16Buffer, 0, 2 * high32count);
if (readNext16 != 2 * high32count)
{
Console.WriteLine("Read error, next16; requested {0}, read {1}", 2 * high32count, readNext16);
}
var next16count = 0;
for (int j = 0; j < high32count; j++)
{
next16count += (int)(next16Buffer[2 * j + 1] == 0 ? ((uint)ushort.MaxValue) + 1 : next16Buffer[2 * j + 1]);
}
if (next08Buffer.Length < 2 * next16count)
{
next08Buffer = new byte[2 * next16count];
}
var readNext08 = next08Reader.Read <byte>(next08Buffer, 0, 2 * next16count);
if (readNext08 != 2 * next16count)
{
Console.WriteLine("Read error, next08; requested {0}, read {1}", 2 * next16count, readNext08);
}
var next08count = 0;
for (int k = 0; k < next16count; k++)
{
next08count += (int)(next08Buffer[2 * k + 1] == 0 ? ((uint)byte.MaxValue) + 1 : next08Buffer[2 * k + 1]);
}
if (last08Buffer.Length < next08count)
{
last08Buffer = new byte[next08count];
}
var readLast08 = last08Reader.Read <byte>(last08Buffer, 0, next08count);
if (readLast08 != next08count)
{
Console.WriteLine("Read error, last08; requested {0}, read {1}", next08count, readLast08);
}
var next16Offset = 0;
var next08Offset = 0;
var last08Offset = 0;
// do work for all these dudes
var xHigh32 = (high32value & 0xFFFF0000) << 0;
var yHigh32 = (high32value & 0x0000FFFF) << 16;
for (int j = 0; j < high32count; j++)
{
var xNext16 = xHigh32 + ((next16Buffer[next16Offset + 2 * j] & 0xFF00) << 0);
var yNext16 = yHigh32 + ((next16Buffer[next16Offset + 2 * j] & 0x00FF) << 8);
next16count = next16Buffer[next16Offset + 2 * j + 1];
if (next16count == 0)
{
next16count = ((int)ushort.MaxValue) + 1;
}
for (int k = 0; k < next16count; k++)
{
var xNext08 = xNext16 + ((next08Buffer[next08Offset + 2 * k] & 0xF0) << 0);
var yNext08 = yNext16 + ((next08Buffer[next08Offset + 2 * k] & 0x0F) << 4);
next08count = next08Buffer[next08Offset + 2 * k + 1];
if (next08count == 0)
{
next08count = ((int)byte.MaxValue) + 1;
}
for (int l = 0; l < next08count; l++)
{
var x = xNext08 + ((last08Buffer[last08Offset + l] & 0xF0) >> 4);
var y = yNext08 + ((last08Buffer[last08Offset + l] & 0x0F) >> 0);
iDegrees[y]++;
oDegrees[x]++;
edgeCount++;
}
last08Offset += next08count;
}
next08Offset += 2 * next16count;
}
}
Console.WriteLine("{1}\tProcessed\t{0} edges", edgeCount, stopwatch.Elapsed);
next16Reader.Dispose();
next08Reader.Dispose();
last08Reader.Dispose();
var graph = new FileGraphScanner(prefix);
graph.ForEach((vertex, degree, offset, neighbors) =>
{
for (int i = 0; i < degree; i++)
{
iDegrees[neighbors[offset + i]]--;
oDegrees[vertex]--;
}
});
var iErrors = iDegrees.Count(x => x > 0);
var oErrors = oDegrees.Count(x => x > 0);
Console.WriteLine("iErrors: {0}\toErrors: {1}", iErrors, oErrors);
Console.WriteLine("{0}\tCounted {1} edges", stopwatch.Elapsed, edgeCount);
}
public static void Scan(this string prefix, Action <uint, uint, int, int, byte[]> perEdgeAction)
{
var high32MetaData = UnbufferedIO.ReadFile <uint>(prefix + "-high32");
var next16Reader = new UnbufferedIO.SequentialReader(prefix + "-next16");
var next08Reader = new UnbufferedIO.SequentialReader(prefix + "-next08");
var last08Reader = new UnbufferedIO.SequentialReader(prefix + "-last08");
var next16Buffer = new ushort[] { };
var next08Buffer = new byte[] { };
var last08Buffer = new byte[] { };
for (int i = 0; i < high32MetaData.Length; i += 2)
{
var high32value = high32MetaData[i];
var high32count = (int)high32MetaData[i + 1];
if (next16Buffer.Length < 2 * high32count)
{
next16Buffer = new ushort[2 * high32count];
}
var readNext16 = next16Reader.Read <ushort>(next16Buffer, 0, 2 * high32count);
if (readNext16 != 2 * high32count)
{
Console.WriteLine("Read error, next16; requested {0}, read {1}", 2 * high32count, readNext16);
}
var next16count = 0;
for (int j = 0; j < high32count; j++)
{
next16count += (int)(next16Buffer[2 * j + 1] == 0 ? ((uint)ushort.MaxValue) + 1 : next16Buffer[2 * j + 1]);
}
if (next08Buffer.Length < 2 * next16count)
{
next08Buffer = new byte[2 * next16count];
}
var readNext08 = next08Reader.Read <byte>(next08Buffer, 0, 2 * next16count);
if (readNext08 != 2 * next16count)
{
Console.WriteLine("Read error, next08; requested {0}, read {1}", 2 * next16count, readNext08);
}
var next08count = 0;
for (int k = 0; k < next16count; k++)
{
next08count += (int)(next08Buffer[2 * k + 1] == 0 ? ((uint)byte.MaxValue) + 1 : next08Buffer[2 * k + 1]);
}
if (last08Buffer.Length < next08count)
{
last08Buffer = new byte[next08count];
}
var readLast08 = last08Reader.Read <byte>(last08Buffer, 0, next08count);
if (readLast08 != next08count)
{
Console.WriteLine("Read error, last08; requested {0}, read {1}", next08count, readLast08);
}
var next16Offset = 0;
var next08Offset = 0;
var last08Offset = 0;
// do work for all these dudes
var xHigh32 = (high32value & 0xFFFF0000) << 0;
var yHigh32 = (high32value & 0x0000FFFF) << 16;
for (int j = 0; j < high32count; j++)
{
var xNext16 = xHigh32 + (uint)((next16Buffer[next16Offset + 2 * j] & 0xFF00) << 0);
var yNext16 = yHigh32 + (uint)((next16Buffer[next16Offset + 2 * j] & 0x00FF) << 8);
next16count = next16Buffer[next16Offset + 2 * j + 1];
if (next16count == 0)
{
next16count = ((int)ushort.MaxValue) + 1;
}
for (int k = 0; k < next16count; k++)
{
var xNext08 = xNext16 + (uint)((next08Buffer[next08Offset + 2 * k] & 0xF0) << 0);
var yNext08 = yNext16 + (uint)((next08Buffer[next08Offset + 2 * k] & 0x0F) << 4);
next08count = next08Buffer[next08Offset + 2 * k + 1];
if (next08count == 0)
{
next08count = ((int)byte.MaxValue) + 1;
}
perEdgeAction(xNext08, yNext08, next08count, last08Offset, last08Buffer);
last08Offset += next08count;
}
next08Offset += 2 * next16count;
}
}
next16Reader.Dispose();
next08Reader.Dispose();
last08Reader.Dispose();
}