public UInt32 PossiblyAddToken(T token, Document doc)
{
UInt32 id;
BTreeSet <Document> docSet;
if (!ValueToId.TryGetValue(token, out id))
{
Update();
id = ++_nextId;
_IdToValue.AddFast(id, token);
_valueToId.Add(token, id);
docSet = new BTreeSet <Document>();
_tokenMap.AddFast(id, docSet);
}
else
{
docSet = _tokenMap[id];
}
UInt32 wordHit;
if (!doc.WordHit.TryGetValue(id, out wordHit))
{
docSet.AddFast(doc);
doc.WordHit.Add(id, 1);
}
else
{
doc.WordHit[id] = ++wordHit;
}
AddToGlobalCount(id, 1);
return(id);
}
static void CreateData()
{
using (SessionNoServer session = new SessionNoServer(s_systemDir))
{
bool dirExist = Directory.Exists(session.SystemDirectory);
if (dirExist)
Directory.Delete(session.SystemDirectory, true); // remove systemDir from prior runs and all its databases.
Directory.CreateDirectory(session.SystemDirectory);
File.Copy(s_licenseDbFile, Path.Combine(session.SystemDirectory, "4.odb"));
DataCache.MaximumMemoryUse = 10000000000; // 10 GB, set this to what fits your case
SessionBase.DefaultCompressPages = PageInfo.compressionKind.LZ4;
session.BeginUpdate();
BTreeMap<Int64, VelocityDbList<Person>> btreeMap = new BTreeMap<Int64, VelocityDbList<Person>>(null, session);
session.Persist(btreeMap);
for (int i = 0; i < 100000; i++)
{
Person p = new Person();
GeoHash geohash = GeoHash.WithBitPrecision(p.Lattitude, p.Longitude);
VelocityDbList<Person> personList;
if (btreeMap.TryGetValue(geohash.LongValue, out personList))
personList.Add(p);
else
{
personList = new VelocityDbList<Person>(1);
//session.Persist(p);
personList.Add(p);
session.Persist(personList);
btreeMap.Add(geohash.LongValue, personList);
}
}
session.Commit();
}
}
public void aCreateDefaultCompareIntKeyIntValue(int number)
{
Oid id;
using (SessionNoServer session = new SessionNoServer(systemDir))
{
Placement place = new Placement((UInt32)number, 1, 1, UInt16.MaxValue, UInt16.MaxValue);
session.Compact();
session.BeginUpdate();
BTreeMap <int, int> bTree = new BTreeMap <int, int>(null, session);
bTree.Persist(place, session);
id = bTree.Oid;
for (int i = 0; i < number; i++)
{
bTree.Add(i, i + 1);
}
bTree.Clear();
for (int i = 0; i < number; i++)
{
bTree.Add(i, i + 1);
}
session.Commit();
}
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginRead();
BTreeMap <int, int> bTree = (BTreeMap <int, int>)session.Open(id);
int count = 0;
int prior = 0;
foreach (KeyValuePair <int, int> pair in bTree)
{
count++;
Assert.True(pair.Key == prior++);
Assert.True(pair.Key == pair.Value - 1);
}
Assert.True(number == count);
session.Commit();
}
}
public void bCreateDefaultCompareIntKeyPersonValue(int number)
{
Oid id;
using (SessionNoServer session = new SessionNoServer(systemDir))
{
Placement place = new Placement((UInt32)number, 1, 1, UInt16.MaxValue, UInt16.MaxValue);
Placement personPlace = new Placement((UInt32)number + 1, 1, 1, UInt16.MaxValue, UInt16.MaxValue);
session.BeginUpdate();
BTreeMap <int, Person> bTree = new BTreeMap <int, Person>(null, session);
bTree.Persist(place, session);
id = bTree.Oid;
Person person;
for (int i = 0; i < number; i++)
{
person = new Person();
person.Persist(personPlace, session);
bTree.Add(i, person);
}
session.Commit();
}
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginRead();
BTreeMap <int, Person> bTree = (BTreeMap <int, Person>)session.Open(id);
int count = 0;
int prior = 0;
foreach (KeyValuePair <int, Person> pair in bTree)
{
count++;
Assert.True(pair.Key == prior++);
Assert.True(pair.Value != null);
}
Assert.True(number == count);
session.Commit();
}
}
static void CreateData()
{
using (SessionNoServer session = new SessionNoServer(s_systemDir))
{
bool dirExist = Directory.Exists(session.SystemDirectory);
if (dirExist)
{
Directory.Delete(session.SystemDirectory, true); // remove systemDir from prior runs and all its databases.
}
Directory.CreateDirectory(session.SystemDirectory);
File.Copy(s_licenseDbFile, Path.Combine(session.SystemDirectory, "4.odb"));
DataCache.MaximumMemoryUse = 10000000000; // 10 GB, set this to what fits your case
SessionBase.DefaultCompressPages = PageInfo.compressionKind.LZ4;
session.BeginUpdate();
BTreeMap <Int64, VelocityDbList <Person> > btreeMap = new BTreeMap <Int64, VelocityDbList <Person> >(null, session);
session.Persist(btreeMap);
for (int i = 0; i < 100000; i++)
{
Person p = new Person();
GeoHash geohash = GeoHash.WithBitPrecision(p.Lattitude, p.Longitude);
VelocityDbList <Person> personList;
if (btreeMap.TryGetValue(geohash.LongValue, out personList))
{
personList.Add(p);
}
else
{
personList = new VelocityDbList <Person>(1);
//session.Persist(p);
personList.Add(p);
session.Persist(personList);
btreeMap.Add(geohash.LongValue, personList);
}
}
session.Commit();
}
}
public void aCreateDefaultCompareIntKeyIntValue(int number)
{
Oid id;
using (SessionNoServer session = new SessionNoServer(systemDir))
{
Placement place = new Placement((UInt32)number, 1, 1, UInt16.MaxValue, UInt16.MaxValue);
session.Compact();
session.BeginUpdate();
BTreeMap<int, int> bTree = new BTreeMap<int, int>(null, session);
bTree.Persist(place, session);
id = bTree.Oid;
for (int i = 0; i < number; i++)
{
bTree.Add(i, i + 1);
}
bTree.Clear();
for (int i = 0; i < number; i++)
{
bTree.Add(i, i + 1);
}
session.Commit();
}
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginRead();
BTreeMap<int, int> bTree = (BTreeMap<int, int>)session.Open(id);
int count = 0;
int prior = 0;
foreach (KeyValuePair<int, int> pair in bTree)
{
count++;
Assert.True(pair.Key == prior++);
Assert.True(pair.Key == pair.Value - 1);
}
Assert.True(number == count);
session.Commit();
}
}
public void bCreateDefaultCompareIntKeyPersonValue(int number)
{
Oid id;
using (SessionNoServer session = new SessionNoServer(systemDir))
{
Placement place = new Placement((UInt32)number, 1, 1, UInt16.MaxValue, UInt16.MaxValue);
Placement personPlace = new Placement((UInt32)number + 1, 1, 1, UInt16.MaxValue, UInt16.MaxValue);
session.BeginUpdate();
BTreeMap<int, Person> bTree = new BTreeMap<int, Person>(null, session);
bTree.Persist(place, session);
id = bTree.Oid;
Person person;
for (int i = 0; i < number; i++)
{
person = new Person();
person.Persist(personPlace, session);
bTree.Add(i, person);
}
session.Commit();
}
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginRead();
BTreeMap<int, Person> bTree = (BTreeMap<int, Person>)session.Open(id);
int count = 0;
int prior = 0;
foreach (KeyValuePair<int, Person> pair in bTree)
{
count++;
Assert.True(pair.Key == prior++);
Assert.True(pair.Value != null);
}
Assert.True(number == count);
session.Commit();
}
}
static void Main(string[] args)
{
long triangles = 0;
try
{
using (SessionNoServer session = new SessionNoServer(systemDir))
{
int numberOfWorkerThreads = -1;
if (args.Length > 0)
{
if (!int.TryParse(args[0], out numberOfWorkerThreads))
{
Console.WriteLine("First parameter is numberOfWorkerThreads which must be an Int32");
}
}
bool useLinq = args.Length > 1;
session.BeginUpdate();
BTreeMapIterator <int, int[]> edgesItr;
int[] edge = null;
BTreeMap <int, int[]> edges = session.AllObjects <BTreeMap <int, int[]> >(false).FirstOrDefault();
if (edges != null)
{
session.Commit();
session.BeginRead();
}
else
{
DatabaseLocation location = session.DatabaseLocations.Default();
edges = new BTreeMap <int, int[]>(null, session, 6000);
session.Persist(edges);
edgesItr = edges.Iterator();
using (StreamReader stream = new StreamReader(edgesInputFile, true))
{
int a;
int b;
string line;
string[] fields;
while ((line = stream.ReadLine()) != null)
{
fields = line.Split(' ');
if (!int.TryParse(fields[0], out a))
{
break;
}
b = int.Parse(fields[1]);
if (a != b)
{
if (edgesItr.CurrentKey() == a || edgesItr.GoTo(a))
{
edge = edgesItr.CurrentValue();
Array.Resize(ref edge, edge.Length + 1);
edge[edge.Length - 1] = b;
edgesItr.ReplaceValue(ref edge); // we need to update the value in the BTreeMap
}
else
{
edge = new int[1];
edge[0] = b;
edges.Add(a, edge);
}
}
}
}
edgesItr = edges.Iterator();
while (edgesItr.MoveNext())
{
edge = edgesItr.CurrentValue();
Array.Sort(edge);
edgesItr.ReplaceValue(ref edge);
}
session.Commit();
session.BeginRead();
}
Console.WriteLine("Number of Nodes found: " + edges.Count);
if (useLinq)
{
Console.WriteLine("Query using LINQ");
}
if (numberOfWorkerThreads > 0)
{
Console.WriteLine("Start of triangle discovery using " + numberOfWorkerThreads + " threads, time is " + DateTime.Now);
}
else if (numberOfWorkerThreads < 0)
{
Console.WriteLine("Start of triangle discovery using system automatically selected number of threads, time is " + DateTime.Now);
}
else
{
Console.WriteLine("Start of triangle discovery using main thread, time is " + DateTime.Now);
}
// Start counting triangles !
if (numberOfWorkerThreads != 0)
{
if (useLinq)
{ // Please help, I have not figured out how to properly do the triangle query using LINQ
int[] edge2values = null;
if (numberOfWorkerThreads > 0)
{
triangles = (from KeyValuePair <int, int[]> edgeFrom in edges
from int edgeTo1 in edgeFrom.Value
where edgeFrom.Key < edgeTo1
from int edgeTo2 in edgeFrom.Value
where edgeFrom.Key < edgeTo2 && edgeTo2 > edgeTo1 && edges.TryGetValue(edgeTo1, out edge2values) && Array.BinarySearch(edge2values, edgeTo2) >= 0
select edge).AsParallel().WithDegreeOfParallelism(numberOfWorkerThreads).Count();
}
else
{
triangles = (from KeyValuePair <int, int[]> edgeFrom in edges
from int edgeTo1 in edgeFrom.Value
where edgeFrom.Key < edgeTo1
from int edgeTo2 in edgeFrom.Value
where edgeFrom.Key <edgeTo2 && edgeTo2> edgeTo1 && edges.TryGetValue(edgeTo1, out edge2values) && Array.BinarySearch(edge2values, edgeTo2) >= 0
select edge).AsParallel().Count();
}
}
else
{
edgesItr = edges.Iterator();
ParallelOptions pOptions = new ParallelOptions();
pOptions.MaxDegreeOfParallelism = numberOfWorkerThreads;
// First type parameter is the type of the source elements
// Second type parameter is the type of the local data (subtotal)
Parallel.ForEach <KeyValuePair <int, int[]>, long>(edges, // source collection
pOptions,
() => 0, // method to initialize the local variable
(pair, loop, subtotal) => // method invoked by the loop on each iteration
{
int nodeId = pair.Key;
int[] nodeTo = pair.Value;
int stop = nodeTo.Length - 1;
int i = stop;
int edgeToStart, edgeTo;
int pos;
while (i >= 0)
{
int[] edgeInfo2;
edgeToStart = nodeTo[i--];
if (nodeId < edgeToStart)
{
if (edges.TryGetValue(edgeToStart, out edgeInfo2))
{
for (int j = stop; j >= i; j--)
{
edgeTo = nodeTo[j];
if (edgeToStart < edgeTo)
{
pos = Array.BinarySearch <int>(edgeInfo2, edgeTo);
if (pos >= 0)
{ // we know this one is connected to edgeInfo.From because it is part of edgeInfo.To
subtotal++;
}
}
else
{
break;
}
}
}
}
else
{
break;
}
}
return(subtotal);
},
// Method to be executed when all loops have completed.
// finalResult is the final value of subtotal. supplied by the ForEach method.
(finalResult) => Interlocked.Add(ref triangles, finalResult));
}
}
else if (useLinq)
{
triangles = queryUsingLINQ(edges);
}
else
{
triangles = discoverTrianglesSingleCore(edges);
}
session.Commit();
}
Console.WriteLine("Number of Triangles found: " + triangles + ", time is " + DateTime.Now);
}
catch (Exception e)
{
System.Console.WriteLine(e);
}
}
static void Main(string[] args)
{
long triangles = 0;
try
{
using (SessionNoServer session = new SessionNoServer(systemDir))
{
int numberOfWorkerThreads = -1;
if (args.Length > 0)
{
if (!int.TryParse(args[0], out numberOfWorkerThreads))
Console.WriteLine("First parameter is numberOfWorkerThreads which must be an Int32");
}
bool useLinq = args.Length > 1;
Placement btreePlace = new Placement(40, 1, 1, 10000, 65500, true);
Placement edgeInfoPlace = new Placement(40, 10000, 1, 10000, 65500, true);
session.BeginUpdate();
BTreeMap<int, int[]> edges;
BTreeMapIterator<int, int[]> edgesItr;
int[] edge = null;
Database edgeDb = session.OpenDatabase(40, false, false);
if (edgeDb != null)
{
session.Commit();
session.BeginRead();
edges = (BTreeMap<int, int[]>)session.Open(40, 1, 1, false);
}
else
{
DatabaseLocation location = session.DatabaseLocations.Default();
//location.CompressPages = false; // no compression should make it faster (?)
session.NewDatabase(40, 395, "Edges");
edges = new BTreeMap<int, int[]>(null, session, 6000);
edges.Persist(btreePlace, session, true);
edgesItr = edges.Iterator();
using (StreamReader stream = new StreamReader(edgesInputFile, true))
{
int a;
int b;
string line;
string[] fields;
while ((line = stream.ReadLine()) != null)
{
fields = line.Split(' ');
if (!int.TryParse(fields[0], out a))
break;
b = int.Parse(fields[1]);
if (a != b)
{
if (edgesItr.CurrentKey() == a || edgesItr.GoTo(a))
{
edge = edgesItr.CurrentValue();
Array.Resize(ref edge, edge.Length + 1);
edge[edge.Length - 1] = b;
edgesItr.ReplaceValue(ref edge); // we need to update the value in the BTreeMap
}
else
{
edge = new int[1];
edge[0] = b;
edges.Add(a, edge);
}
}
}
}
edgesItr = edges.Iterator();
while (edgesItr.MoveNext())
{
edge = edgesItr.CurrentValue();
Array.Sort(edge);
edgesItr.ReplaceValue(ref edge);
}
session.Commit();
session.BeginRead();
}
Console.WriteLine("Number of Nodes found: " + edges.Count);
if (useLinq)
Console.WriteLine("Query using LINQ");
if (numberOfWorkerThreads > 0)
Console.WriteLine("Start of triangle discovery using " + numberOfWorkerThreads + " threads, time is " + DateTime.Now);
else if (numberOfWorkerThreads < 0)
Console.WriteLine("Start of triangle discovery using system automatically selected number of threads, time is " + DateTime.Now);
else
Console.WriteLine("Start of triangle discovery using main thread, time is " + DateTime.Now);
// Start counting triangles !
if (numberOfWorkerThreads != 0)
{
if (useLinq)
{ // Please help, I have not figured out how to properly do the triangle query using LINQ
int[] edge2values = null;
if (numberOfWorkerThreads > 0)
triangles = (from KeyValuePair<int, int[]> edgeFrom in edges
from int edgeTo1 in edgeFrom.Value
where edgeFrom.Key < edgeTo1
from int edgeTo2 in edgeFrom.Value
where edgeFrom.Key < edgeTo2 && edgeTo2 > edgeTo1 && edges.TryGetValue(edgeTo1, out edge2values) && Array.BinarySearch(edge2values, edgeTo2) >= 0
select edge).AsParallel().WithDegreeOfParallelism(numberOfWorkerThreads).Count();
else
triangles = (from KeyValuePair<int, int[]> edgeFrom in edges
from int edgeTo1 in edgeFrom.Value
where edgeFrom.Key < edgeTo1
from int edgeTo2 in edgeFrom.Value
where edgeFrom.Key < edgeTo2 && edgeTo2 > edgeTo1 && edges.TryGetValue(edgeTo1, out edge2values) && Array.BinarySearch(edge2values, edgeTo2) >= 0
select edge).AsParallel().Count();
}
else
{
edgesItr = edges.Iterator();
ParallelOptions pOptions = new ParallelOptions();
pOptions.MaxDegreeOfParallelism = numberOfWorkerThreads;
// First type parameter is the type of the source elements
// Second type parameter is the type of the local data (subtotal)
Parallel.ForEach<KeyValuePair<int, int[]>, long>(edges, // source collection
pOptions,
() => 0, // method to initialize the local variable
(pair, loop, subtotal) => // method invoked by the loop on each iteration
{
int nodeId = pair.Key;
int[] nodeTo = pair.Value;
int stop = nodeTo.Length - 1;
int i = stop;
int edgeToStart, edgeTo;
int pos;
while (i >= 0)
{
int[] edgeInfo2;
edgeToStart = nodeTo[i--];
if (nodeId < edgeToStart)
{
if (edges.TryGetValue(edgeToStart, out edgeInfo2))
{
for (int j = stop; j >= i; j--)
{
edgeTo = nodeTo[j];
if (edgeToStart < edgeTo)
{
pos = Array.BinarySearch<int>(edgeInfo2, edgeTo);
if (pos >= 0)
{ // we know this one is connected to edgeInfo.From because it is part of edgeInfo.To
subtotal++;
}
}
else
break;
}
}
}
else
break;
}
return subtotal;
},
// Method to be executed when all loops have completed.
// finalResult is the final value of subtotal. supplied by the ForEach method.
(finalResult) => Interlocked.Add(ref triangles, finalResult));
}
}
else if (useLinq)
triangles = queryUsingLINQ(edges);
else
triangles = discoverTrianglesSingleCore(edges);
session.Commit();
}
Console.WriteLine("Number of Triangles found: " + triangles);
}
catch (Exception e)
{
System.Console.WriteLine(e);
}
}
