static HilbertCurve()
{
Rotated = new bool[256][];
Flipped = new bool[256][];
Transformed = new ushort[256][];
Inverted = new byte[256 * 256 * 2];
for (uint i = 0; i < 256; i++)
{
Rotated[i] = new bool[256];
Flipped[i] = new bool[256];
Transformed[i] = new ushort[256];
for (uint j = 0; j < 256; j++)
{
var transform = (UInt16)HilbertCurve.xy2dAlt(8, i, j, ref Rotated[i][j], ref Flipped[i][j]);
Transformed[i][j] = transform;
Inverted[2 * transform + 0] = (byte)i;
Inverted[2 * transform + 1] = (byte)j;
}
}
}
private static void WriteTransformed(ulong[] edges,
System.IO.BinaryWriter high32Stream,
System.IO.BinaryWriter next16Stream,
System.IO.BinaryWriter next08Stream,
System.IO.BinaryWriter last08Stream)
{
Console.WriteLine("Writing {0} edges", edges.Length);
if (edges.Length == 0)
{
return;
}
uint x = 0;
uint y = 0;
// recover x and y, to interleave their bits
HilbertCurve.d2xyByte(edges[0], out x, out y);
// interleave each bit pattern we want to collect together
var high32Prior = (uint)((x & 0xFFFF0000) + ((y & 0xFFFF0000) >> 16));
var next16Prior = (ushort)((x & 0x0000FF00) + ((y & 0x0000FF00) >> 8));
var next08Prior = (byte)((x & 0x000000F0) + ((y & 0x000000F0) >> 4));
// count distinct subsequent bit patterns
var high32Count = (uint)1;
var next16Count = (uint)1;
var next08Count = (uint)1;
last08Stream.Write((byte)(((x & 0x0F) << 4) + (y & 0x0F)));
for (int i = 1; i < edges.Length; i++)
{
// recover x and y, to interleave their bits
HilbertCurve.d2xyByte(edges[i], out x, out y);
var high32 = (uint)((x & 0xFFFF0000) + ((y & 0xFFFF0000) >> 16));
var next16 = (ushort)((x & 0x0000FF00) + ((y & 0x0000FF00) >> 8));
var next08 = (byte)((x & 0x000000F0) + ((y & 0x000000F0) >> 4));
// changes in bits call for writing out
var high32Change = high32 != high32Prior;
var next16Change = high32Change || (next16 != next16Prior);
var next08Change = next16Change || (next08 != next08Prior);
// write high 32, reset high32count
if (high32Change)
{
high32Stream.Write(high32Prior);
high32Stream.Write(high32Count);
high32Prior = high32;
high32Count = 0;
}
// write next 16, reset nex16count, bump high32count
if (next16Change)
{
next16Stream.Write(next16Prior);
next16Stream.Write((ushort)next16Count);
high32Count++;
next16Prior = next16;
next16Count = 0;
}
// write next 08, reset next08count, bump next16count
if (next08Change)
{
next08Stream.Write(next08Prior);
next08Stream.Write((byte)next08Count);
next16Count++;
next08Prior = next08;
next08Count = 0;
}
// always write the last byte, bump next08count
last08Stream.Write((byte)(((x & 0x0F) << 4) + (y & 0x0F)));
next08Count++;
}
// flush final metadata records
next08Stream.Write(next08Prior);
next08Stream.Write(next08Count);
next16Stream.Write(next16Prior);
next16Stream.Write(next16Count);
high32Stream.Write(high32Prior);
high32Stream.Write(high32Count);
}
public static void ConvertToHilbert(this string prefix)
{
var graph = new FileGraphScanner(prefix);
var High32 = new Dictionary <UInt32, UInt32>();
graph.ForEach((vertex, degree, offset, neighbors) =>
{
//if (vertex < 1000000)
{
for (int i = 0; i < degree; i++)
{
var neighbor = neighbors[offset + i];
var high32 = (uint)(HilbertCurve.xy2dByte((uint)vertex, (uint)neighbor) >> 32);
if (!High32.ContainsKey(high32))
{
High32.Add(high32, 0);
}
High32[high32] = High32[high32] + 1;
}
}
});
Console.WriteLine("Assesed prefix sizes");
var pairs = High32.OrderBy(x => x.Key).ToArray();
var edgeCount = 0L;
var lowIndex = (uint)0;
var high32Stream = new System.IO.BinaryWriter(System.IO.File.OpenWrite(prefix + "-high32"));
var next16Stream = new System.IO.BinaryWriter(System.IO.File.OpenWrite(prefix + "-next16"));
var next08Stream = new System.IO.BinaryWriter(System.IO.File.OpenWrite(prefix + "-next08"));
var last08Stream = new System.IO.BinaryWriter(System.IO.File.OpenWrite(prefix + "-last08"));
for (uint i = 0; i < pairs.Length; i++)
{
// if we would have too many edges, do work
if (edgeCount + pairs[i].Value > 1 << 29)
{
var edges = new ulong[edgeCount];
var cursor = 0;
graph.ForEach((vertex, degree, offset, neighbors) =>
{
//if (vertex < 1000000)
{
for (int j = 0; j < degree; j++)
{
var transform = HilbertCurve.xy2dByte((uint)vertex, (uint)neighbors[offset + j]);
if ((transform >> 32) >= pairs[lowIndex].Key && (transform >> 32) < pairs[i].Key)
{
edges[cursor++] = transform;
}
}
}
});
if (cursor != edges.Length)
{
Console.WriteLine("Somehow read the wrong number of edges {0} vs {1}", cursor, edges.Length);
}
Array.Sort(edges);
Console.WriteLine("About to process {0} high32 blocks", i - lowIndex);
// traverse edges in order and write out when interesting things happen.
WriteTransformed(edges, high32Stream, next16Stream, next08Stream, last08Stream);
edgeCount = 0;
lowIndex = i;
}
edgeCount += pairs[i].Value;
}
var edges2 = new ulong[edgeCount];
var cursor2 = 0;
graph.ForEach((vertex, degree, offset, neighbors) =>
{
//if (vertex < 1000000)
{
for (int j = 0; j < degree; j++)
{
var transform = HilbertCurve.xy2dByte((uint)vertex, (uint)neighbors[offset + j]);
if ((transform >> 32) >= pairs[lowIndex].Key)
{
edges2[cursor2++] = transform;
}
}
}
});
Array.Sort(edges2);
if (cursor2 != edges2.Length)
{
Console.WriteLine("Somehow read the wrong number of edges {0} vs {1}", cursor2, edges2.Length);
}
// traverse edges in order and write out when interesting things happen.
WriteTransformed(edges2, high32Stream, next16Stream, next08Stream, last08Stream);
high32Stream.Close();
next16Stream.Close();
next08Stream.Close();
last08Stream.Close();
}
public static void Test(int tests)
{
var random = new Random(0);
var testxs = new uint[tests];
var testys = new uint[tests];
for (int i = 0; i < tests; i++)
{
testxs[i] = (uint)random.Next(Int32.MaxValue);
testys[i] = (uint)random.Next(Int32.MaxValue);
}
var results1 = new long[tests];
var results2 = new ulong[tests];
for (int i = 0; i < results1.Length; i++)
{
results1[i] = i;
}
for (uint i = 0; i < results2.Length; i++)
{
results2[i] = i;
}
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
stopwatch.Restart();
for (int i = 0; i < tests; i++)
{
results1[i] = HilbertCurve.xy2d(32, (int)testxs[i], (int)testys[i]);
}
Console.WriteLine("Slow encode:\t{0:0.00}ns/record", stopwatch.ElapsedMilliseconds / (tests / 1000000.0));
stopwatch.Restart();
for (int i = 0; i < tests; i++)
{
int x = 0, y = 0;
HilbertCurve.d2xy(32, results1[i], out x, out y);
}
Console.WriteLine("Slow decode:\t{0:0.00}ns/record", stopwatch.ElapsedMilliseconds / (tests / 1000000.0));
stopwatch.Restart();
for (int i = 0; i < tests; i++)
{
results2[i] = HilbertCurve.xy2dByte(testxs[i], testys[i]);
}
Console.WriteLine("Fast encode:\t{0:0.00}ns/record", stopwatch.ElapsedMilliseconds / (tests / 1000000.0));
stopwatch.Restart();
for (int i = 0; i < tests; i++)
{
uint x = 0, y = 0;
HilbertCurve.d2xyByte((ulong)results2[i], out x, out y);
}
Console.WriteLine("Fast decode:\t{0:0.00}ns/record", stopwatch.ElapsedMilliseconds / (tests / 1000000.0));
for (int i = 0; i < tests; i++)
{
if (results1[i] != (long)results2[i])
{
Console.WriteLine("Error! ({0}, {1}) -> [ {2}, {3} ]", testxs[i], testys[i], results1[i], results2[i]);
}
uint x = 0;
uint y = 0;
HilbertCurve.d2xyByte((ulong)results1[i], out x, out y);
if (x != testxs[i] || y != testys[i])
{
Console.WriteLine("Error!!! {4} -> ({0}, {2}) != ({1}, {3})", x, testxs[i], y, testys[i], results1[i]);
Console.ReadLine();
}
}
}
static void ExecuteSingleThreaded(string[] args, string dataDir)
{
string ukFile = Path.Combine(dataDir, @"uk-2007-05");
string twitterFile = Path.Combine(dataDir, @"twitter_rv.bin");
string livejournalFile = Path.Combine(dataDir, @"livejournal.bin");
if (args.Length < 3)
{
throw new Exception("Three arguments required: system, algorithm, dataset");
}
var algorithm = args[1];
var dataset = args[2];
#region file conversions
if (algorithm == "convert" && dataset == "twitter")
{
SingleThreaded.ConvertGraph(twitterFile);
}
if (algorithm == "partition" && dataset == "twitter")
{
SingleThreaded.PartitionGraph(twitterFile, 4, (s, t) => (s & 1) + 2 * (t & 1), dataDir + @"twitter-part-{0}-of-{1}");
}
if (algorithm == "transpose" && dataset == "twitter")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.TransposeGraph(dataDir + @"twitterfollowers\twitter_rv.bin", 65000000);
Console.WriteLine(stopwatch.Elapsed);
}
#endregion
#region hilbert layout
if (algorithm == "hilbertlayout" && dataset == "twitter")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
uint nodes = 0;
var names = new uint[65000000];
for (int i = 0; i < names.Length; i++)
{
names[i] = uint.MaxValue;
}
var idegrees = new int[65000000];
var odegrees = new int[65000000];
var edges = 0L;
SingleThreaded.ScanGraph(twitterFile, (vertex, degree, offset, neighbors) =>
{
if (names[vertex] == uint.MaxValue)
{
names[vertex] = nodes++;
}
for (int i = 0; i < degree; i++)
{
if (names[neighbors[offset + i]] == uint.MaxValue)
{
names[neighbors[offset + i]] = nodes++;
}
}
edges += degree;
});
Console.WriteLine("{2}\tNodes: {0}\tEdges: {1}", nodes, edges, stopwatch.Elapsed);
// allocate enough space for all the edges.
var hilbertTransformed = new uint[edges];
var counts = new uint[1 << 20];
SingleThreaded.ScanGraph(twitterFile, (vertex, degree, offset, neighbors) =>
{
for (int i = 0; i < degree; i++)
{
counts[HilbertCurve.xy2dByte(names[vertex], names[neighbors[offset + i]]) >> 32]++;
}
});
Console.WriteLine("{0}\tHilbert regions sized", stopwatch.Elapsed);
for (int i = 1; i < counts.Length; i++)
{
counts[i + 1] += counts[i];
}
for (int i = counts.Length - 1; i > 0; i--)
{
counts[i] = counts[i] - 1;
}
counts[0] = 0;
var Trie = new BufferTrie <uint>(20, (array, offset, length) =>
{
for (int i = offset; i < offset + length; i++)
{
hilbertTransformed[counts[array[i].Index]++] = array[i].Value;
}
});
for (int i = counts.Length - 1; i > 0; i--)
{
counts[i] = counts[i] - 1;
}
counts[0] = 0;
var buffer = new BufferTrie <uint> .Pair[5000000];
SingleThreaded.ScanGraph(twitterFile, (vertex, degree, offset, neighbors) =>
{
for (int i = 0; i < degree; i++)
{
var result = HilbertCurve.xy2dByte(names[vertex], names[neighbors[offset + i]]);
buffer[i] = new BufferTrie <uint> .Pair((int)(result >> 32), (uint)(result & 0xFFFF));
}
Trie.Insert(buffer, 0, degree);
});
Trie.Flush();
Console.WriteLine("{0}\tEdges partitioned", stopwatch.Elapsed);
using (var upper = new System.IO.BinaryWriter(System.IO.File.OpenWrite("twitter-hilbert-upper")))
{
for (uint i = 0; i < counts.Length - 1; i++)
{
if (counts[i] < counts[i + 1])
{
uint x = 0, y = 0;
HilbertCurve.d2xyByte((i << 32), out x, out y);
upper.Write(x);
upper.Write(y);
upper.Write(counts[i + 1] - counts[i]);
}
}
}
using (var lower = new System.IO.BinaryWriter(System.IO.File.OpenWrite("twitter-hilbert-lower")))
{
for (uint i = 0; i < counts.Length - 1; i++)
{
Array.Sort(hilbertTransformed, (int)counts[i], (int)(counts[i + 1] - counts[i]));
for (uint j = counts[i]; j < counts[i + 1]; j++)
{
uint x = 0, y = 0;
HilbertCurve.d2xyByte((i << 32) + hilbertTransformed[j], out x, out y);
lower.Write((UInt16)(x & 0xFFFF));
lower.Write((UInt16)(y & 0xFFFF));
}
}
}
}
#endregion
#region hilbert pagerank
if (algorithm == "hilbertpagerank" && dataset == "livejournal")
{
unsafe
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var nodes = (uint)42000000;
#if UseLargePages
var srcRanks = LargePages.AllocateFloats(nodes);
var dstRanks = LargePages.AllocateFloats(nodes);
#else
var srcRanks = new float[nodes];
var dstRanks = new float[nodes];
#endif
for (int i = 0; i < nodes; i++)
{
srcRanks[i] = 1.0f;
}
SingleThreaded.HilbertPagerank(@"livejournal-hilbert", dstRanks, srcRanks, nodes, 0.85f);
Console.WriteLine(stopwatch.Elapsed);
}
}
if (algorithm == "hilbertpagerank" && dataset == "twitter")
{
unsafe
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var nodes = (uint)42000000;
#if UseLargePages
var srcRanks = LargePages.AllocateFloats(nodes);
var dstRanks = LargePages.AllocateFloats(nodes);
#else
var srcRanks = new float[nodes];
var dstRanks = new float[nodes];
#endif
for (int i = 0; i < nodes; i++)
{
srcRanks[i] = 1.0f;
}
SingleThreaded.HilbertPagerank(@"twitter-hilbert", dstRanks, srcRanks, nodes, 0.85f);
Console.WriteLine(stopwatch.Elapsed);
}
}
if (algorithm == "hilbertpagerank" && dataset == "uk-2007-05")
{
unsafe
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var nodes = (uint)106000000;
var srcRanks = LargePages.AllocateFloats(nodes);
var dstRanks = LargePages.AllocateFloats(nodes);
for (int i = 0; i < nodes; i++)
{
srcRanks[i] = 1.0f;
}
SingleThreaded.MultiHilbertPagerank(@"uk-2007-05-hilbert", dstRanks, srcRanks, nodes, 0.85f);
Console.WriteLine(stopwatch.Elapsed);
}
}
#endregion
#region hilbert union find
if (algorithm == "hilbertunionfind" && dataset == "twitter")
{
unsafe
{
var nodes = (uint)42000000;
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.HilbertUnionFind2(@"twitter-hilbert", nodes);
Console.WriteLine(stopwatch.Elapsed);
}
}
if (algorithm == "hilbertunionfind" && dataset == "uk-2007-05")
{
unsafe
{
var nodes = (uint)106000000;
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.MultiHilbertCC(@"uk-2007-05-hilbert", nodes);
Console.WriteLine(stopwatch.Elapsed);
}
}
if (algorithm == "hilbertunionfind" && dataset == "livejournal")
{
unsafe
{
var nodes = (uint)42000000;
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.HilbertUnionFind(@"livejournal-hilbert-upper", "livejournal-hilbert-lower", nodes);
Console.WriteLine(stopwatch.Elapsed);
}
}
#endregion
#region page rank
if (algorithm == "pagerank" && dataset == "uk-2007-05")
{
unsafe
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var nodes = (uint)106000000;
#if UseLargePages
var srcRanks = LargePages.AllocateFloats(nodes);
var dstRanks = LargePages.AllocateFloats(nodes);
#else
var srcRanks = new float[nodes];
var dstRanks = new float[nodes];
#endif
SingleThreaded.PageRankFromDisk(ukFile, dstRanks, srcRanks, nodes, 0.85f);
Console.WriteLine(stopwatch.Elapsed);
}
}
if (algorithm == "pagerank" && dataset == "twitter")
{
unsafe
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var nodes = (uint)65000000;
#if UseLargePages
var srcRanks = LargePages.AllocateFloats(nodes);
var dstRanks = LargePages.AllocateFloats(nodes);
#else
var srcRanks = new float[nodes];
var dstRanks = new float[nodes];
#endif
SingleThreaded.PageRankFromDisk(twitterFile, dstRanks, srcRanks, nodes, 0.85f);
Console.WriteLine(stopwatch.Elapsed);
}
}
if (algorithm == "pagerank" && dataset == "livejournal")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
unsafe
{
var nodes = (uint)65000000;
#if UseLargePages
var srcRanks = LargePages.AllocateFloats(nodes);
var dstRanks = LargePages.AllocateFloats(nodes);
#else
var srcRanks = new float[nodes];
var dstRanks = new float[nodes];
#endif
for (int i = 0; i < 20; i++)
{
SingleThreaded.PageRankStep(livejournalFile, dstRanks, srcRanks, nodes, 0.85f);
Console.WriteLine("{0}\tIteration {1}", stopwatch.Elapsed, i);
}
}
Console.WriteLine(stopwatch.Elapsed);
}
#endregion
#region connected components
if (algorithm == "connectedcomponents" && dataset == "twitter")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.ConnectedComponents(twitterFile, 65000000);
Console.WriteLine(stopwatch.Elapsed);
}
if (algorithm == "connectedcomponents" && dataset == "uk-2007-05")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.ClumsyCC(ukFile, 106000000);
Console.WriteLine(stopwatch.Elapsed);
}
if (algorithm == "connectedcomponents" && dataset == "livejournal")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.ConnectedComponents(livejournalFile, 6500000);
Console.WriteLine(stopwatch.Elapsed);
}
#endregion
#region maximal independent set
if (algorithm == "maximalindependentset" && dataset == "twitter")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.MaximalIndependentSet(twitterFile, 65000000);
Console.WriteLine(stopwatch.Elapsed);
}
if (algorithm == "maximalindependentset" && dataset == "livejournal")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.MaximalIndependentSet(livejournalFile, 6500000);
Console.WriteLine(stopwatch.Elapsed);
}
#endregion
}