public unsafe static void ClumsyCC(string filename, uint nodes)
{
var labels = LargePages.AllocateInts(nodes);
for (int i = 0; i < nodes; i++)
{
labels[i] = i;
}
var oldSum = Int64.MaxValue;
var newSum = 0L;
for (int i = 0; i < nodes; i++)
{
newSum += labels[i];
}
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
while (newSum < oldSum)
{
var startTime = stopwatch.Elapsed;
// apply per-vertex pagerank logic across the graph
ScanGraph(filename, (vertex, degree, offset, neighbors) =>
{
var label = labels[vertex];
for (int i = 0; i < degree; i++)
{
label = Math.Min(label, labels[neighbors[offset + i]]);
}
labels[vertex] = label;
for (int i = 0; i < degree; i++)
{
labels[neighbors[offset + i]] = label;
}
});
oldSum = newSum;
newSum = 0L;
for (int i = 0; i < nodes; i++)
{
newSum += labels[i];
}
Console.WriteLine("{0}", stopwatch.Elapsed - startTime);
}
}
public static unsafe void HilbertPagerank(string filename, float *a, float *b, uint nodes, float reset)
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var degrees = LargePages.AllocateInts(nodes);
//var degrees = new int[nodes];
var metadata = UnbufferedIO.ReadFile <uint>(filename + "-upper");
var edges = UnbufferedIO.ReadFile <uint>(filename + "-lower");
Console.WriteLine("{2}\tRead header of {0} blocks, for {1} edges", metadata.Length / 3, edges.Length, stopwatch.Elapsed);
ScanGraphHilbert(metadata, edges, (srcUpper, tgtUpper, offset, count, edgeArray) =>
{
for (var j = offset; j < offset + count; j++)
{
degrees[srcUpper + (edges[j] & 0xFFFF)]++;
}
});
Console.WriteLine("{0}\tDegrees calculated", stopwatch.Elapsed);
for (int iteration = 0; iteration < 20; iteration++)
{
var startTime = stopwatch.Elapsed;
for (int i = 0; i < nodes; i++)
{
b[i] = (1.0f - reset) * a[i] / degrees[i];
a[i] = reset;
}
ScanGraphHilbert(metadata, edges, (sourceUpper, targetUpper, offset, count, edgeArray) =>
{
for (var j = offset; j < offset + count; j++)
{
a[targetUpper + (edges[j] >> 16)] += b[sourceUpper + (edges[j] & 0xFFFF)];
}
});
Console.WriteLine("{0}\tIteration {1} in {2}", stopwatch.Elapsed, iteration, stopwatch.Elapsed - startTime);
}
}
public static unsafe void HilbertPagerankFromDisk(string filename, float[] a, float[] b, uint nodes, float reset)
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
#if UseLargePages
var degrees = LargePages.AllocateInts(nodes);
#else
var degrees = new int[nodes];
#endif
ScanGraphHilbert(filename, (srcUpper, tgtUpper, offset, count, edges) =>
{
for (int i = 0; i < count; i++)
{
degrees[srcUpper + (edges[offset + i] >> 16)]++;
}
});
Console.WriteLine("{0}\tDegrees calculated", stopwatch.Elapsed);
for (int iteration = 0; iteration < 20; iteration++)
{
var startTime = stopwatch.Elapsed;
ScanGraphHilbert(filename, (srcUpper, tgtUpper, offset, count, edges) =>
{
for (int i = 0; i < count; i++)
{
a[tgtUpper + (edges[offset + i] >> 16)] += b[srcUpper + (edges[offset + i] & 0xFFFF)];
}
});
for (int i = 0; i < nodes; i++)
{
a[i] = a[i] / degrees[i];
}
Console.WriteLine("{0}\tIteration {1} in {2}", stopwatch.Elapsed, iteration, stopwatch.Elapsed - startTime);
}
}
public static unsafe void MultiHilbertPagerank(string filename, float *a, float *b, uint nodes, float reset)
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var degrees = LargePages.AllocateInts(nodes);
var scanner = new MultiHilbertScanner(filename);
scanner.Scan((xUpper, yUpper, count, offset, edges) =>
{
for (var j = offset; j < offset + count; j++)
{
degrees[xUpper + ((edges[j] & 0xF0) >> 4)]++;
}
});
Console.WriteLine("{0}\tDegrees calculated", stopwatch.Elapsed);
for (int iteration = 0; iteration < 20; iteration++)
{
var startTime = stopwatch.Elapsed;
for (int i = 0; i < nodes; i++)
{
b[i] = (1.0f - reset) * a[i] / degrees[i];
a[i] = reset;
}
scanner.Scan((xUpper, yUpper, count, offset, edges) =>
{
for (var j = offset; j < offset + count; j++)
{
a[yUpper + (edges[j] & 0x0F)] += b[xUpper + ((edges[j] & 0xF0) >> 4)];
}
});
Console.WriteLine("{0}\tIteration {1} in {2}", stopwatch.Elapsed, iteration, stopwatch.Elapsed - startTime);
}
}
public static unsafe void HilbertUnionFind2(string filename, uint nodes)
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
#if UseLargePages
var roots = LargePages.AllocateInts(nodes);
var ranks = LargePages.AllocateBytes(nodes);
#else
var roots = new int[nodes];
var ranks = new byte[nodes];
#endif
for (int i = 0; i < nodes; i++)
{
roots[i] = i;
}
ScanGraphHilbert(filename, (sourceUpper, targetUpper, offset, count, edges) =>
{
for (int j = 0; j < count; j++)
{
var source = (int)(sourceUpper + (edges[offset + j] & 0xFFFF));
var target = (int)(targetUpper + (edges[offset + j] >> 16));
if (source != target)
{
while (source != roots[source])
{
source = roots[source];
}
while (target != roots[target])
{
target = roots[target];
}
// union(source, target)
if (source != target)
{
// there may be a tie in ranks
if (ranks[source] == ranks[target])
{
// break ties towards lower ids
if (source < target)
{
ranks[source]++;
roots[target] = source;
}
else
{
ranks[target]++;
roots[source] = target;
}
}
else
{
// attatch lower rank to higher
if (ranks[source] < ranks[target])
{
roots[source] = target;
}
else
{
roots[target] = source;
}
}
}
}
}
});
// path compress all vertices to roots.
for (int i = 0; i < nodes; i++)
{
while (roots[i] != roots[roots[i]])
{
roots[i] = roots[roots[i]];
}
}
var counter = 0;
for (int i = 0; i < nodes; i++)
{
if (roots[i] != i)
{
counter++;
}
}
Console.WriteLine("Edges found: {0}", counter);
}
public static unsafe void HilbertUnionFind(string filenameUpper, string filenameLower, uint nodes)
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
using (var upper = System.IO.File.OpenRead(filenameUpper))
{
using (var lower = System.IO.File.OpenRead(filenameLower))
{
var bytes = new byte[upper.Length];
upper.Read(bytes, 0, bytes.Length);
var uppers = new int[bytes.Length / 4];
Buffer.BlockCopy(bytes, 0, uppers, 0, bytes.Length);
var maxEdges = 0;
for (int i = 0; i < uppers.Length; i += 3)
{
maxEdges = Math.Max(maxEdges, uppers[i + 2]);
}
Console.WriteLine("{1}\tRead header of {0} blocks", uppers.Length / 3, stopwatch.Elapsed);
bytes = new byte[8 * maxEdges];
var lowers = new UInt16[2 * maxEdges];
var edges = new Int32[2 * maxEdges];
#if UseLargePages
var roots = LargePages.AllocateInts(nodes);
var ranks = LargePages.AllocateBytes(nodes);
#else
var roots = new int[nodes];
var ranks = new byte[nodes];
#endif
for (int i = 0; i < nodes; i++)
{
roots[i] = i;
}
for (int i = 0; i < uppers.Length; i += 3)
{
var read = lower.Read(bytes, 0, 4 * uppers[i + 2]);
Buffer.BlockCopy(bytes, 0, lowers, 0, read);
var sourceUpper = uppers[i + 0] << 16;
var targetUpper = uppers[i + 1] << 16;
var count = uppers[i + 2];
for (int j = 0; j < count; j++)
{
edges[2 * j + 0] = roots[sourceUpper + lowers[2 * j + 0]];
edges[2 * j + 1] = roots[targetUpper + lowers[2 * j + 1]];
}
for (int j = 0; j < count; j++)
{
var source = edges[2 * j + 0];
var target = edges[2 * j + 1];
if (source != target)
{
while (source != roots[source])
{
source = roots[source];
}
while (target != roots[target])
{
target = roots[target];
}
// union(source, target)
if (source != target)
{
// there may be a tie in ranks
if (ranks[source] == ranks[target])
{
// break ties towards lower ids
if (source < target)
{
ranks[source]++;
roots[target] = source;
}
else
{
ranks[target]++;
roots[source] = target;
}
}
else
{
// attatch lower rank to higher
if (ranks[source] < ranks[target])
{
roots[source] = target;
}
else
{
roots[target] = source;
}
}
}
}
}
}
// path compress all vertices to roots.
for (int i = 0; i < nodes; i++)
{
while (roots[i] != roots[roots[i]])
{
roots[i] = roots[roots[i]];
}
}
var counter = 0;
for (int i = 0; i < nodes; i++)
{
if (roots[i] != i)
{
counter++;
}
}
Console.WriteLine("Edges found: {0}", counter);
}
}
}
public static unsafe void MultiHilbertCC(string filename, uint nodes)
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var roots = LargePages.AllocateInts(nodes);
var ranks = LargePages.AllocateBytes(nodes);
for (int i = 0; i < nodes; i++)
{
roots[i] = i;
}
var scanner = new MultiHilbertScanner(filename);
scanner.Scan((xUpper, yUpper, count, offset, edges) =>
{
for (int j = 0; j < count; j++)
{
var source = (int)(xUpper + ((edges[offset + j] & 0xF0) >> 4));
var target = (int)(yUpper + ((edges[offset + j] & 0x0F) >> 0));
if (source != target)
{
while (source != roots[source])
{
source = roots[source];
}
while (target != roots[target])
{
target = roots[target];
}
// union(source, target)
if (source != target)
{
// there may be a tie in ranks
if (ranks[source] == ranks[target])
{
// break ties towards lower ids
if (source < target)
{
ranks[source]++;
roots[target] = source;
}
else
{
ranks[target]++;
roots[source] = target;
}
}
else
{
// attatch lower rank to higher
if (ranks[source] < ranks[target])
{
roots[source] = target;
}
else
{
roots[target] = source;
}
}
}
}
}
});
// path compress all vertices to roots.
for (int i = 0; i < nodes; i++)
{
while (roots[i] != roots[roots[i]])
{
roots[i] = roots[roots[i]];
}
}
var counter = 0;
for (int i = 0; i < nodes; i++)
{
if (roots[i] != i)
{
counter++;
}
}
Console.WriteLine("{1}\tEdges found: {0}", counter, stopwatch.Elapsed);
}
public unsafe static void ConnectedComponents(string filename, uint nodes)
{
#if UseLargePages
var roots = LargePages.AllocateInts(nodes);
var ranks = LargePages.AllocateBytes(nodes);
#else
var roots = new int[nodes];
var ranks = new byte[nodes];
#endif
for (int i = 0; i < nodes; i++)
{
roots[i] = i;
}
// apply per-vertex union-find logic across the graph
ScanGraph(filename, (vertex, degree, offset, neighbors) =>
{
// find(vertex)
var source = roots[vertex];
while (source != roots[source])
{
source = roots[source];
}
for (int i = 0; i < degree; i++)
{
// find(neighbors[i])
var target = roots[neighbors[offset + i]];
while (target != roots[target])
{
target = roots[target];
}
// union(source, target)
if (source != target)
{
// there may be a tie in ranks
if (ranks[source] == ranks[target])
{
// break ties towards lower ids
if (source < target)
{
ranks[source]++;
roots[target] = source;
}
else
{
ranks[target]++;
roots[source] = target;
source = target;
}
}
else
{
// attatch lower rank to higher
if (ranks[source] < ranks[target])
{
roots[source] = target;
source = target;
}
else
{
roots[target] = source;
}
}
}
}
});
var counter = 0;
for (int i = 0; i < nodes; i++)
{
if (roots[i] != i)
{
counter++;
}
}
Console.WriteLine("Edges found: {0}", counter);
}
static void ExecuteSingleThreaded(string[] args, string dataDir)
{
string ukFile = Path.Combine(dataDir, @"uk-2007-05");
string twitterFile = Path.Combine(dataDir, @"twitter_rv.bin");
string livejournalFile = Path.Combine(dataDir, @"livejournal.bin");
if (args.Length < 3)
{
throw new Exception("Three arguments required: system, algorithm, dataset");
}
var algorithm = args[1];
var dataset = args[2];
#region file conversions
if (algorithm == "convert" && dataset == "twitter")
{
SingleThreaded.ConvertGraph(twitterFile);
}
if (algorithm == "partition" && dataset == "twitter")
{
SingleThreaded.PartitionGraph(twitterFile, 4, (s, t) => (s & 1) + 2 * (t & 1), dataDir + @"twitter-part-{0}-of-{1}");
}
if (algorithm == "transpose" && dataset == "twitter")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.TransposeGraph(dataDir + @"twitterfollowers\twitter_rv.bin", 65000000);
Console.WriteLine(stopwatch.Elapsed);
}
#endregion
#region hilbert layout
if (algorithm == "hilbertlayout" && dataset == "twitter")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
uint nodes = 0;
var names = new uint[65000000];
for (int i = 0; i < names.Length; i++)
{
names[i] = uint.MaxValue;
}
var idegrees = new int[65000000];
var odegrees = new int[65000000];
var edges = 0L;
SingleThreaded.ScanGraph(twitterFile, (vertex, degree, offset, neighbors) =>
{
if (names[vertex] == uint.MaxValue)
{
names[vertex] = nodes++;
}
for (int i = 0; i < degree; i++)
{
if (names[neighbors[offset + i]] == uint.MaxValue)
{
names[neighbors[offset + i]] = nodes++;
}
}
edges += degree;
});
Console.WriteLine("{2}\tNodes: {0}\tEdges: {1}", nodes, edges, stopwatch.Elapsed);
// allocate enough space for all the edges.
var hilbertTransformed = new uint[edges];
var counts = new uint[1 << 20];
SingleThreaded.ScanGraph(twitterFile, (vertex, degree, offset, neighbors) =>
{
for (int i = 0; i < degree; i++)
{
counts[HilbertCurve.xy2dByte(names[vertex], names[neighbors[offset + i]]) >> 32]++;
}
});
Console.WriteLine("{0}\tHilbert regions sized", stopwatch.Elapsed);
for (int i = 1; i < counts.Length; i++)
{
counts[i + 1] += counts[i];
}
for (int i = counts.Length - 1; i > 0; i--)
{
counts[i] = counts[i] - 1;
}
counts[0] = 0;
var Trie = new BufferTrie <uint>(20, (array, offset, length) =>
{
for (int i = offset; i < offset + length; i++)
{
hilbertTransformed[counts[array[i].Index]++] = array[i].Value;
}
});
for (int i = counts.Length - 1; i > 0; i--)
{
counts[i] = counts[i] - 1;
}
counts[0] = 0;
var buffer = new BufferTrie <uint> .Pair[5000000];
SingleThreaded.ScanGraph(twitterFile, (vertex, degree, offset, neighbors) =>
{
for (int i = 0; i < degree; i++)
{
var result = HilbertCurve.xy2dByte(names[vertex], names[neighbors[offset + i]]);
buffer[i] = new BufferTrie <uint> .Pair((int)(result >> 32), (uint)(result & 0xFFFF));
}
Trie.Insert(buffer, 0, degree);
});
Trie.Flush();
Console.WriteLine("{0}\tEdges partitioned", stopwatch.Elapsed);
using (var upper = new System.IO.BinaryWriter(System.IO.File.OpenWrite("twitter-hilbert-upper")))
{
for (uint i = 0; i < counts.Length - 1; i++)
{
if (counts[i] < counts[i + 1])
{
uint x = 0, y = 0;
HilbertCurve.d2xyByte((i << 32), out x, out y);
upper.Write(x);
upper.Write(y);
upper.Write(counts[i + 1] - counts[i]);
}
}
}
using (var lower = new System.IO.BinaryWriter(System.IO.File.OpenWrite("twitter-hilbert-lower")))
{
for (uint i = 0; i < counts.Length - 1; i++)
{
Array.Sort(hilbertTransformed, (int)counts[i], (int)(counts[i + 1] - counts[i]));
for (uint j = counts[i]; j < counts[i + 1]; j++)
{
uint x = 0, y = 0;
HilbertCurve.d2xyByte((i << 32) + hilbertTransformed[j], out x, out y);
lower.Write((UInt16)(x & 0xFFFF));
lower.Write((UInt16)(y & 0xFFFF));
}
}
}
}
#endregion
#region hilbert pagerank
if (algorithm == "hilbertpagerank" && dataset == "livejournal")
{
unsafe
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var nodes = (uint)42000000;
#if UseLargePages
var srcRanks = LargePages.AllocateFloats(nodes);
var dstRanks = LargePages.AllocateFloats(nodes);
#else
var srcRanks = new float[nodes];
var dstRanks = new float[nodes];
#endif
for (int i = 0; i < nodes; i++)
{
srcRanks[i] = 1.0f;
}
SingleThreaded.HilbertPagerank(@"livejournal-hilbert", dstRanks, srcRanks, nodes, 0.85f);
Console.WriteLine(stopwatch.Elapsed);
}
}
if (algorithm == "hilbertpagerank" && dataset == "twitter")
{
unsafe
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var nodes = (uint)42000000;
#if UseLargePages
var srcRanks = LargePages.AllocateFloats(nodes);
var dstRanks = LargePages.AllocateFloats(nodes);
#else
var srcRanks = new float[nodes];
var dstRanks = new float[nodes];
#endif
for (int i = 0; i < nodes; i++)
{
srcRanks[i] = 1.0f;
}
SingleThreaded.HilbertPagerank(@"twitter-hilbert", dstRanks, srcRanks, nodes, 0.85f);
Console.WriteLine(stopwatch.Elapsed);
}
}
if (algorithm == "hilbertpagerank" && dataset == "uk-2007-05")
{
unsafe
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var nodes = (uint)106000000;
var srcRanks = LargePages.AllocateFloats(nodes);
var dstRanks = LargePages.AllocateFloats(nodes);
for (int i = 0; i < nodes; i++)
{
srcRanks[i] = 1.0f;
}
SingleThreaded.MultiHilbertPagerank(@"uk-2007-05-hilbert", dstRanks, srcRanks, nodes, 0.85f);
Console.WriteLine(stopwatch.Elapsed);
}
}
#endregion
#region hilbert union find
if (algorithm == "hilbertunionfind" && dataset == "twitter")
{
unsafe
{
var nodes = (uint)42000000;
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.HilbertUnionFind2(@"twitter-hilbert", nodes);
Console.WriteLine(stopwatch.Elapsed);
}
}
if (algorithm == "hilbertunionfind" && dataset == "uk-2007-05")
{
unsafe
{
var nodes = (uint)106000000;
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.MultiHilbertCC(@"uk-2007-05-hilbert", nodes);
Console.WriteLine(stopwatch.Elapsed);
}
}
if (algorithm == "hilbertunionfind" && dataset == "livejournal")
{
unsafe
{
var nodes = (uint)42000000;
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.HilbertUnionFind(@"livejournal-hilbert-upper", "livejournal-hilbert-lower", nodes);
Console.WriteLine(stopwatch.Elapsed);
}
}
#endregion
#region page rank
if (algorithm == "pagerank" && dataset == "uk-2007-05")
{
unsafe
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var nodes = (uint)106000000;
#if UseLargePages
var srcRanks = LargePages.AllocateFloats(nodes);
var dstRanks = LargePages.AllocateFloats(nodes);
#else
var srcRanks = new float[nodes];
var dstRanks = new float[nodes];
#endif
SingleThreaded.PageRankFromDisk(ukFile, dstRanks, srcRanks, nodes, 0.85f);
Console.WriteLine(stopwatch.Elapsed);
}
}
if (algorithm == "pagerank" && dataset == "twitter")
{
unsafe
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var nodes = (uint)65000000;
#if UseLargePages
var srcRanks = LargePages.AllocateFloats(nodes);
var dstRanks = LargePages.AllocateFloats(nodes);
#else
var srcRanks = new float[nodes];
var dstRanks = new float[nodes];
#endif
SingleThreaded.PageRankFromDisk(twitterFile, dstRanks, srcRanks, nodes, 0.85f);
Console.WriteLine(stopwatch.Elapsed);
}
}
if (algorithm == "pagerank" && dataset == "livejournal")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
unsafe
{
var nodes = (uint)65000000;
#if UseLargePages
var srcRanks = LargePages.AllocateFloats(nodes);
var dstRanks = LargePages.AllocateFloats(nodes);
#else
var srcRanks = new float[nodes];
var dstRanks = new float[nodes];
#endif
for (int i = 0; i < 20; i++)
{
SingleThreaded.PageRankStep(livejournalFile, dstRanks, srcRanks, nodes, 0.85f);
Console.WriteLine("{0}\tIteration {1}", stopwatch.Elapsed, i);
}
}
Console.WriteLine(stopwatch.Elapsed);
}
#endregion
#region connected components
if (algorithm == "connectedcomponents" && dataset == "twitter")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.ConnectedComponents(twitterFile, 65000000);
Console.WriteLine(stopwatch.Elapsed);
}
if (algorithm == "connectedcomponents" && dataset == "uk-2007-05")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.ClumsyCC(ukFile, 106000000);
Console.WriteLine(stopwatch.Elapsed);
}
if (algorithm == "connectedcomponents" && dataset == "livejournal")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.ConnectedComponents(livejournalFile, 6500000);
Console.WriteLine(stopwatch.Elapsed);
}
#endregion
#region maximal independent set
if (algorithm == "maximalindependentset" && dataset == "twitter")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.MaximalIndependentSet(twitterFile, 65000000);
Console.WriteLine(stopwatch.Elapsed);
}
if (algorithm == "maximalindependentset" && dataset == "livejournal")
{
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
SingleThreaded.MaximalIndependentSet(livejournalFile, 6500000);
Console.WriteLine(stopwatch.Elapsed);
}
#endregion
}
