public void aaaFakeLicenseDatabase()
{
Assert.Throws <NoValidVelocityDBLicenseFoundException>(() =>
{
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
Database database;
License license = new License("Mats", 1, null, null, null, 99999, DateTime.MaxValue, 9999, 99, 9999);
Placement placer = new Placement(License.PlaceInDatabase, 1, 1, 1);
license.Persist(placer, session);
for (uint i = 10; i < 20; i++)
{
database = session.NewDatabase(i);
Assert.NotNull(database);
}
session.Commit();
File.Copy(Path.Combine(systemDir, "20.odb"), Path.Combine(systemDir, "4.odb"));
session.BeginUpdate();
for (uint i = 21; i < 30; i++)
{
database = session.NewDatabase(i);
Assert.NotNull(database);
}
session.RegisterClass(typeof(BTreeSet <int>));
session.RegisterClass(typeof(BTreeSet <long>));
session.RegisterClass(typeof(BTreeSet <DateTime>));
session.RegisterClass(typeof(BTreeSet <double>));
session.RegisterClass(typeof(BTreeMap <string, double>));
session.Commit();
}
});
}
public void aaaFakeLicenseDatabase()
{
Assert.Throws<NoValidVelocityDBLicenseFoundException>(() =>
{
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
Database database;
License license = new License("Mats", 1, null, null, null, 99999, DateTime.MaxValue, 9999, 99, 9999);
Placement placer = new Placement(License.PlaceInDatabase, 1, 1, 1);
license.Persist(placer, session);
for (uint i = 10; i < 20; i++)
{
database = session.NewDatabase(i);
Assert.NotNull(database);
}
session.Commit();
File.Copy(Path.Combine(systemDir, "20.odb"), Path.Combine(systemDir, "4.odb"));
session.BeginUpdate();
for (uint i = 21; i < 30; i++)
{
database = session.NewDatabase(i);
Assert.NotNull(database);
}
session.RegisterClass(typeof(VelocityDbSchema.Samples.Sample1.Person));
Graph g = new Graph(session);
session.Persist(g);
session.Commit();
}
});
}
public void aaaFakeLicenseDatabase()
{
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
Database database;
License license = new License("Mats", 1, null, null, null, 99999, DateTime.MaxValue, 9999, 99, 9999);
Placement placer = new Placement(License.PlaceInDatabase, 1, 1, 1);
license.Persist(placer, session);
for (uint i = 10; i < 20; i++)
{
database = session.NewDatabase(i);
Assert.NotNull(database);
}
session.Commit();
File.Copy(Path.Combine(systemDir, "20.odb"), Path.Combine(systemDir, "4.odb"));
session.BeginUpdate();
for (uint i = 21; i < 30; i++)
{
database = session.NewDatabase(i);
Assert.NotNull(database);
}
session.RegisterClass(typeof(VelocityDbSchema.Samples.Sample1.Person));
Graph g = new Graph(session);
session.Persist(g);
session.Commit();
}
}
public void License()
{
using (SessionNoServer session = new SessionNoServer(systemDir))
{
Database database;
session.BeginUpdate();
for (uint i = 60000000; i < 60000010; i++)
{
database = session.NewDatabase(i);
Assert.NotNull(database);
}
session.Commit();
session.BeginUpdate();
for (uint i = 60000000; i < 60000010; i++)
{
database = session.OpenDatabase(i);
Assert.NotNull(database);
}
session.Commit();
session.BeginUpdate();
for (uint i = 60000000; i < 60000010; i++)
{
database = session.OpenDatabase(i);
session.DeleteDatabase(database);
}
session.Commit();
}
}
public void TwoUpdaters1()
{
Assert.Throws<OptimisticLockingFailed>(() =>
{
UInt64 id;
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
Man man = new Man();
man.Persist(session, man);
id = man.Id;
session.Commit();
session.BeginUpdate();
man.Age = ++man.Age;
Database db = session.NewDatabase(3567);
using (SessionNoServer session2 = new SessionNoServer(systemDir))
{
session2.BeginUpdate();
Man man2 = (Man)session2.Open(id);
Assert.Less(man2.Age, man.Age);
man2.Age = ++man.Age;
session2.Commit();
}
session.DeleteDatabase(db);
session.Commit(); // OptimisticLockingFailed here
}
});
}
public void aCreateDatabases()
{
Database database;
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
for (uint i = 50000000; i < 50001000; i++)
{
database = session.NewDatabase(i);
Assert.NotNull(database);
}
session.Commit();
}
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
for (uint i = 50000000; i < 50001000; i++)
{
database = session.OpenDatabase(i);
Assert.NotNull(database);
}
session.Commit();
}
}
public void CreateLocationSameHost()
{
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
DatabaseLocation remoteLocation = new DatabaseLocation(Dns.GetHostName(), location2Dir, locationStartDbNum, locationEndDbNum, session, PageInfo.compressionKind.LZ4, 0);
remoteLocation = session.NewLocation(remoteLocation);
Database database = session.NewDatabase(remoteLocation.StartDatabaseNumber);
Assert.NotNull(database);
session.Commit();
}
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
Database database = session.OpenDatabase(locationStartDbNum, false);
Assert.NotNull(database);
session.DeleteDatabase(database);
session.Commit();
}
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
foreach (DatabaseLocation loc in session.DatabaseLocations)
{
Console.WriteLine(loc.ToStringDetails(session, false));
}
session.DeleteLocation(session.DatabaseLocations.LocationForDb(locationStartDbNum));
foreach (DatabaseLocation loc in session.DatabaseLocations)
{
Console.WriteLine(loc.ToStringDetails(session, false));
}
session.Commit();
}
}
public void TwoUpdaters1()
{
Assert.Throws <OptimisticLockingFailed>(() =>
{
UInt64 id;
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
Man man = new Man();
man.Persist(session, man);
id = man.Id;
session.Commit();
session.BeginUpdate();
man.Age = ++man.Age;
Database db = session.NewDatabase(3567);
using (SessionNoServer session2 = new SessionNoServer(systemDir))
{
session2.BeginUpdate();
Man man2 = (Man)session2.Open(id);
Assert.Less(man2.Age, man.Age);
man2.Age = ++man.Age;
session2.Commit();
}
session.DeleteDatabase(db);
session.Commit(); // OptimisticLockingFailed here
session.Verify();
}
});
}
public void MultipleThreadsAdding()
{
bool doClearAll = SessionBase.ClearAllCachedObjectsWhenDetectingUpdatedDatabase;
SessionBase.ClearAllCachedObjectsWhenDetectingUpdatedDatabase = false;
try
{
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
session.RegisterClass(typeof(AutoPlacement)); // build in type but not yet registered as a one
session.RegisterClass(typeof(ObservableList <int>));
session.RegisterClass(typeof(Dokument));
UInt32 dbNum = session.DatabaseNumberOf(typeof(Dokument));
Database db = session.OpenDatabase(dbNum, false, false);
if (db == null)
{
db = session.NewDatabase(dbNum, 0, typeof(Dokument).ToGenericTypeString());
}
Dokument doc = new Dokument();
session.Persist(doc);
session.Commit();
}
using (ServerClientSessionShared sharedReadSession = new ServerClientSessionShared(systemDir))
{
sharedReadSession.BeginRead();
Parallel.ForEach(Enumerable.Range(1, 3), (num) => LockConflict(sharedReadSession));
}
}
finally
{
SessionBase.ClearAllCachedObjectsWhenDetectingUpdatedDatabase = doClearAll;
}
}
public void CreateLocationSameHost()
{
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
DatabaseLocation remoteLocation = new DatabaseLocation(Dns.GetHostName(), location2Dir, locationStartDbNum, locationEndDbNum, session, PageInfo.compressionKind.LZ4, 0);
remoteLocation = session.NewLocation(remoteLocation);
Database database = session.NewDatabase(remoteLocation.StartDatabaseNumber);
Assert.NotNull(database);
session.Commit();
}
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
Database database = session.OpenDatabase(locationStartDbNum, false);
Assert.NotNull(database);
session.DeleteDatabase(database);
session.Commit();
}
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
foreach (DatabaseLocation loc in session.DatabaseLocations)
Console.WriteLine(loc.ToStringDetails(session, false));
session.DeleteLocation(session.DatabaseLocations.LocationForDb(locationStartDbNum));
foreach (DatabaseLocation loc in session.DatabaseLocations)
Console.WriteLine(loc.ToStringDetails(session, false));
session.Commit();
}
}
public void aCreateDatabases()
{
Database database;
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
for (uint i = 50000000; i < 50001000; i++)
{
database = session.NewDatabase(i);
Assert.NotNull(database);
}
session.Commit();
}
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
for (uint i = 50000000; i < 50001000; i++)
{
database = session.OpenDatabase(i);
Assert.NotNull(database);
}
session.Commit();
}
}
public void aSyncNewDatabases()
{
using (SessionBase session = new SessionNoServer(s_sync1))
{
session.EnableSyncByTrackingChanges = true;
using (var trans = session.BeginUpdate())
{
for (uint i = 10; i < 50; i++)
{
var database = session.NewDatabase(i);
Assert.NotNull(database);
}
session.Commit();
}
}
using (SessionBase readFromSession = new SessionNoServer(s_sync1))
{
using (SessionBase updateSession = new SessionNoServer(s_sync2))
{
updateSession.SyncWith(readFromSession);
}
}
using (SessionBase readFromSession = new SessionNoServer(s_sync1))
{
readFromSession.BeginRead();
using (SessionBase updateSession = new SessionNoServer(s_sync2))
{
using (var trans = updateSession.BeginRead())
{
Assert.AreEqual(updateSession.OpenAllDatabases().Count, readFromSession.OpenAllDatabases().Count - 1); // - 1 due to additional change tracking databases in original
}
}
}
}
public void aSyncNewDatabases()
{
using (SessionBase session = new SessionNoServer(s_sync1))
{
session.EnableSyncByTrackingChanges = true;
using (var trans = session.BeginUpdate())
{
for (uint i = 10; i < 50; i++)
{
var database = session.NewDatabase(i);
Assert.NotNull(database);
}
session.Commit();
}
}
using (SessionBase readFromSession = new SessionNoServer(s_sync1))
{
using (SessionBase updateSession = new SessionNoServer(s_sync2))
{
updateSession.SyncWith(readFromSession);
}
}
using (SessionBase readFromSession = new SessionNoServer(s_sync1))
{
readFromSession.BeginRead();
using (SessionBase updateSession = new SessionNoServer(s_sync2))
{
using (var trans = updateSession.BeginRead())
{
Assert.AreEqual(updateSession.OpenAllDatabases().Count, readFromSession.OpenAllDatabases().Count - 1); // - 1 due to additional change tracking databases in original
}
}
}
}
static void importEntireWikipedia()
{
const ushort btreeNodeSize = 10000;
Console.WriteLine(DateTime.Now.ToString() + ", start importing Wikipedia text");
//System.Xml.Schema.XmlSchema docSchema;
//using (System.Xml.XmlTextReader schemaReader = new System.Xml.XmlTextReader("c:\\export-0_5.xsd"))
//{
// docSchema = System.Xml.Schema.XmlSchema.Read(schemaReader, ValidationCallBack);
// }
int docCount = 0;
using (SessionNoServer session = new SessionNoServer(s_systemDir, 5000, false, false, CacheEnum.No)) // turn of page and object caching
{
Console.WriteLine("Running with databases in directory: " + session.SystemDirectory);
//GCSettings.LatencyMode = GCLatencyMode.Batch;// try to keep the WeakIOptimizedPersistableReference objects around longer
Placement documentPlacement = new Placement(Document.PlaceInDatabase, 1003, 1, 500, 1000, false, false, 1000, false);
Placement contentPlacement = new Placement(Document.PlaceInDatabase, 1, 1, 500, UInt16.MaxValue, false, false, 1, false);
XmlComment xmlComment;
XmlElement xmlElement;
XmlEntity xmlEntity;
XmlText xmlText;
XmlWhitespace xmlWhitespace;
session.BeginUpdate();
File.Copy(s_licenseDbFile, System.IO.Path.Combine(session.SystemDirectory, "4.odb"), true);
// register all database schema classes used by the application in advance to avoid lock conflict later in parallell indexing
session.RegisterClass(typeof(Repository));
session.RegisterClass(typeof(IndexRoot));
session.RegisterClass(typeof(Document));
session.RegisterClass(typeof(Lexicon));
session.RegisterClass(typeof(DocumentText));
session.RegisterClass(typeof(Word));
session.RegisterClass(typeof(WordGlobal));
session.RegisterClass(typeof(WordHit));
session.RegisterClass(typeof(BTreeSet <Document>));
session.RegisterClass(typeof(OidShort));
session.RegisterClass(typeof(BTreeMap <Word, WordHit>));
session.RegisterClass(typeof(HashCodeComparer <Word>));
session.RegisterClass(typeof(BTreeSetOidShort <Word>));
session.RegisterClass(typeof(BTreeMapOidShort <Word, WordHit>));
Database db = session.OpenDatabase(IndexRoot.PlaceInDatabase, false, false);
if (db != null)
{
outputSomeInfo(session);
session.Abort();
return;
}
session.NewDatabase(IndexRoot.PlaceInDatabase, 0, "IndexRoot");
session.NewDatabase(Lexicon.PlaceInDatabase, 0, "Lexicon");
session.NewDatabase(Repository.PlaceInDatabase, 0, "Repository");
for (UInt32 i = 40; i <= 186; i++)
{
session.NewDatabase(i, 512, "Document"); // pre allocate 146 Document databases presized to 512MB each
}
//session.SetTraceDbActivity(Lexicon.PlaceInDatabase);
//session.SetTraceAllDbActivity();
XmlDocument xmlDocument = new XmlDocument("enwiki-latest-pages-articles.xml");
IndexRoot indexRoot = new IndexRoot(btreeNodeSize, session);
indexRoot.Persist(session, indexRoot, true);
Document doc = null;
bool titleElement = false;
bool pageText = false;
UInt32 currentDocumentDatabaseNum = documentPlacement.StartDatabaseNumber;
using (FileStream fs = new FileStream(s_wikipediaXmlFile, FileMode.Open))
{
//using (GZipStream zipStream = new GZipStream(fs, CompressionMode.Decompress)) // if input was a .gz file
{
using (System.Xml.XmlTextReader textReader = new System.Xml.XmlTextReader(fs))
{
while (textReader.Read())
{
System.Xml.XmlNodeType nodeType = textReader.NodeType;
switch (nodeType)
{
case System.Xml.XmlNodeType.Attribute:
break;
case System.Xml.XmlNodeType.CDATA:
break;
case System.Xml.XmlNodeType.Comment:
xmlComment = new XmlComment(textReader.Value, xmlDocument);
break;
case System.Xml.XmlNodeType.Document:
break;
case System.Xml.XmlNodeType.DocumentFragment:
break;
case System.Xml.XmlNodeType.DocumentType:
break;
case System.Xml.XmlNodeType.Element:
xmlElement = new XmlElement(textReader.Prefix, textReader.LocalName, textReader.NamespaceURI, xmlDocument);
if (textReader.LocalName == "title")
{
titleElement = true;
}
else if (textReader.LocalName == "text")
{
pageText = true;
}
break;
case System.Xml.XmlNodeType.EndElement:
if (textReader.LocalName == "title" && doc != null)
{
titleElement = false;
}
else if (textReader.LocalName == "text" && doc != null)
{
pageText = false;
}
break;
case System.Xml.XmlNodeType.EndEntity:
break;
case System.Xml.XmlNodeType.Entity:
xmlEntity = new XmlEntity(textReader.LocalName, xmlDocument);
break;
case System.Xml.XmlNodeType.EntityReference:
break;
case System.Xml.XmlNodeType.None:
break;
case System.Xml.XmlNodeType.Notation:
break;
case System.Xml.XmlNodeType.ProcessingInstruction:
break;
case System.Xml.XmlNodeType.SignificantWhitespace:
break;
case System.Xml.XmlNodeType.Text:
xmlText = new XmlText(textReader.Value, xmlDocument);
if (titleElement)
{
doc = new Document(textReader.Value, indexRoot, session);
doc.Persist(documentPlacement, session, true);
if (doc.DatabaseNumber != currentDocumentDatabaseNum)
{
session.FlushUpdates(session.OpenDatabase(currentDocumentDatabaseNum));
Console.WriteLine("Database: " + currentDocumentDatabaseNum + " is completed, done importing article " + docCount + " number of lines: " + textReader.LineNumber);
currentDocumentDatabaseNum = doc.DatabaseNumber;
}
//doc.Page.Database.Name = doc.Name;
}
else if (doc != null && pageText)
{
#if DEBUGx
Console.WriteLine(doc.Name + " line: " + textReader.LineNumber);
#endif
//if (textReader.LineNumber > 1000000)
//{
// session.Commit();
// return;
//}
DocumentText content = new DocumentText(textReader.Value, doc);
if (doc.DatabaseNumber != contentPlacement.TryDatabaseNumber)
{
contentPlacement = new Placement(doc.DatabaseNumber, (ushort)contentPlacement.StartPageNumber, 1, contentPlacement.MaxObjectsPerPage, contentPlacement.MaxPagesPerDatabase, false, false, 1, false);
}
content.Persist(contentPlacement, session, false);
Debug.Assert(content.DatabaseNumber == doc.DatabaseNumber);
doc.Content = content;
indexRoot.repository.documentSet.AddFast(doc);
if (++docCount % 1000000 == 0)
{
//session.Commit(false); // skip recovery check, we do it in BeginUpdate which is enough
Console.WriteLine("Done importing article " + docCount + " number of lines: " + textReader.LineNumber);
//session.BeginUpdate();
}
}
break;
case System.Xml.XmlNodeType.Whitespace:
xmlWhitespace = new XmlWhitespace(textReader.Value, xmlDocument);
break;
case System.Xml.XmlNodeType.XmlDeclaration:
break;
}
;
}
Console.WriteLine("Finished importing article " + docCount + " number of lines: " + textReader.LineNumber);
}
}
}
session.Commit();
}
Console.WriteLine(DateTime.Now.ToString() + ", done importing Wikipedia text");
}
public MainWindow()
{
const ushort btreeNodeSize = 5000;
GCSettings.LatencyMode = GCLatencyMode.Batch;// try to keep the WeakIOptimizedPersistableReference objects around longer
dataGridList = new List<DataGrid>();
dataTableList = new List<DataTable>();
InitializeComponent();
session = new SessionNoServer(s_systemDir);
Placement placerIndexRoot = new Placement(IndexRoot.PlaceInDatabase);
session.BeginUpdate();
Console.WriteLine("Running with databases in directory: " + session.SystemDirectory);
File.Copy(s_licenseDbFile, Path.Combine(session.SystemDirectory, "4.odb"), true);
IndexRoot indexRoot;
Database db = session.OpenDatabase(IndexRoot.PlaceInDatabase, false, false);
if (db == null)
{
session.NewDatabase(IndexRoot.PlaceInDatabase, 0, "IndexRoot");
session.NewDatabase(Lexicon.PlaceInDatabase, 0, "Lexicon");
session.NewDatabase(Document.PlaceInDatabase, 0, "Document");
session.NewDatabase(Repository.PlaceInDatabase, 0, "Repository");
session.NewDatabase(DocumentText.PlaceInDatabase, 0, "DocumentText");
session.NewDatabase(Word.PlaceInDatabase, 0, "Word");
indexRoot = new IndexRoot(btreeNodeSize, session);
if (Directory.Exists(s_booksDir))
{
string[] directoryTextFiles = Directory.GetFiles(s_booksDir, "*.txt");
foreach (string fileName in directoryTextFiles)
{
listBoxPagesToAdd.Items.Add(fileName);
}
}
else
{
wordMinCt.Text = 1.ToString();
listBoxPagesToAdd.Items.Add("http://www.VelocityDB.com/");
// other database products
listBoxPagesToAdd.Items.Add("https://foundationdb.com/");
listBoxPagesToAdd.Items.Add("http://www.oracle.com/us/products/database/index.html");
listBoxPagesToAdd.Items.Add("http://www-01.ibm.com/software/data/db2/");
listBoxPagesToAdd.Items.Add("http://www.versant.com/");
listBoxPagesToAdd.Items.Add("http://web.progress.com/en/objectstore/");
listBoxPagesToAdd.Items.Add("https://www.mongodb.org/");
listBoxPagesToAdd.Items.Add("http://cassandra.apache.org/");
listBoxPagesToAdd.Items.Add("http://www.sybase.com/");
listBoxPagesToAdd.Items.Add("http://www.mcobject.com/perst");
listBoxPagesToAdd.Items.Add("http://www.marklogic.com/what-is-marklogic/");
listBoxPagesToAdd.Items.Add("http://hamsterdb.com/");
listBoxPagesToAdd.Items.Add("http://www.firebirdsql.org/");
listBoxPagesToAdd.Items.Add("http://www.h2database.com/");
listBoxPagesToAdd.Items.Add("http://www.oracle.com/technology/products/berkeley-db");
listBoxPagesToAdd.Items.Add("http://www.scimore.com/");
listBoxPagesToAdd.Items.Add("http://www.stsdb.com/");
listBoxPagesToAdd.Items.Add("http://www.sqlite.org/about.html");
listBoxPagesToAdd.Items.Add("http://www.mysql.com/products/enterprise/techspec.html");
listBoxPagesToAdd.Items.Add("http://www.objectivity.com");
listBoxPagesToAdd.Items.Add("http://vistadb.net/");
listBoxPagesToAdd.Items.Add("http://www.google.com/search?q=object+database&sourceid=ie7&rls=com.microsoft:en-us:IE-SearchBox&ie=&oe=");
}
indexRoot.Persist(session, indexRoot);
}
else
indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1));
if (indexRoot.repository.documentSet.Count > 0)
{
List<Document> docs = indexRoot.repository.documentSet.ToList<Document>().Take(50).ToList<Document>();
inDbListBox.ItemsSource = docs;
}
updateDataGrids(indexRoot);
session.Commit();
//verify();
}
static void importEntireWikipedia()
{
const ushort btreeNodeSize = 10000;
Console.WriteLine(DateTime.Now.ToString() + ", start importing Wikipedia text");
//System.Xml.Schema.XmlSchema docSchema;
//using (System.Xml.XmlTextReader schemaReader = new System.Xml.XmlTextReader("c:\\export-0_5.xsd"))
//{
// docSchema = System.Xml.Schema.XmlSchema.Read(schemaReader, ValidationCallBack);
// }
int docCount = 0;
using (SessionNoServer session = new SessionNoServer(s_systemDir, 5000, false, false, CacheEnum.No)) // turn of page and object caching
{
Console.WriteLine("Running with databases in directory: " + session.SystemDirectory);
//GCSettings.LatencyMode = GCLatencyMode.Batch;// try to keep the WeakIOptimizedPersistableReference objects around longer
Placement documentPlacement = new Placement(Document.PlaceInDatabase, 1003, 1, 500, 1000, false, false, 1000, false);
Placement contentPlacement = new Placement(Document.PlaceInDatabase, 1, 1, 500, UInt16.MaxValue, false, false, 1, false);
XmlComment xmlComment;
XmlElement xmlElement;
XmlEntity xmlEntity;
XmlText xmlText;
XmlWhitespace xmlWhitespace;
session.BeginUpdate();
File.Copy(s_licenseDbFile, System.IO.Path.Combine(session.SystemDirectory, "4.odb"), true);
// register all database schema classes used by the application in advance to avoid lock conflict later in parallell indexing
session.RegisterClass(typeof(Repository));
session.RegisterClass(typeof(IndexRoot));
session.RegisterClass(typeof(Document));
session.RegisterClass(typeof(Lexicon));
session.RegisterClass(typeof(DocumentText));
session.RegisterClass(typeof(Word));
session.RegisterClass(typeof(WordGlobal));
session.RegisterClass(typeof(WordHit));
session.RegisterClass(typeof(BTreeSet<Document>));
session.RegisterClass(typeof(OidShort));
session.RegisterClass(typeof(BTreeMap<Word, WordHit>));
session.RegisterClass(typeof(HashCodeComparer<Word>));
session.RegisterClass(typeof(BTreeSetOidShort<Word>));
session.RegisterClass(typeof(BTreeMapOidShort<Word, WordHit>));
Database db = session.OpenDatabase(IndexRoot.PlaceInDatabase, false, false);
if (db != null)
{
outputSomeInfo(session);
session.Abort();
return;
}
session.NewDatabase(IndexRoot.PlaceInDatabase, 0, "IndexRoot");
session.NewDatabase(Lexicon.PlaceInDatabase, 0, "Lexicon");
session.NewDatabase(Repository.PlaceInDatabase, 0, "Repository");
for (UInt32 i = 40; i <= 186; i++)
{
session.NewDatabase(i, 512, "Document"); // pre allocate 146 Document databases presized to 512MB each
}
//session.SetTraceDbActivity(Lexicon.PlaceInDatabase);
//session.SetTraceAllDbActivity();
XmlDocument xmlDocument = new XmlDocument("enwiki-latest-pages-articles.xml");
IndexRoot indexRoot = new IndexRoot(btreeNodeSize, session);
indexRoot.Persist(session, indexRoot, true);
Document doc = null;
bool titleElement = false;
bool pageText = false;
UInt32 currentDocumentDatabaseNum = documentPlacement.StartDatabaseNumber;
using (FileStream fs = new FileStream(s_wikipediaXmlFile, FileMode.Open))
{
//using (GZipStream zipStream = new GZipStream(fs, CompressionMode.Decompress)) // if input was a .gz file
{
using (System.Xml.XmlTextReader textReader = new System.Xml.XmlTextReader(fs))
{
while (textReader.Read())
{
System.Xml.XmlNodeType nodeType = textReader.NodeType;
switch (nodeType)
{
case System.Xml.XmlNodeType.Attribute:
break;
case System.Xml.XmlNodeType.CDATA:
break;
case System.Xml.XmlNodeType.Comment:
xmlComment = new XmlComment(textReader.Value, xmlDocument);
break;
case System.Xml.XmlNodeType.Document:
break;
case System.Xml.XmlNodeType.DocumentFragment:
break;
case System.Xml.XmlNodeType.DocumentType:
break;
case System.Xml.XmlNodeType.Element:
xmlElement = new XmlElement(textReader.Prefix, textReader.LocalName, textReader.NamespaceURI, xmlDocument);
if (textReader.LocalName == "title")
titleElement = true;
else if (textReader.LocalName == "text")
pageText = true;
break;
case System.Xml.XmlNodeType.EndElement:
if (textReader.LocalName == "title" && doc != null)
titleElement = false;
else if (textReader.LocalName == "text" && doc != null)
pageText = false;
break;
case System.Xml.XmlNodeType.EndEntity:
break;
case System.Xml.XmlNodeType.Entity:
xmlEntity = new XmlEntity(textReader.LocalName, xmlDocument);
break;
case System.Xml.XmlNodeType.EntityReference:
break;
case System.Xml.XmlNodeType.None:
break;
case System.Xml.XmlNodeType.Notation:
break;
case System.Xml.XmlNodeType.ProcessingInstruction:
break;
case System.Xml.XmlNodeType.SignificantWhitespace:
break;
case System.Xml.XmlNodeType.Text:
xmlText = new XmlText(textReader.Value, xmlDocument);
if (titleElement)
{
doc = new Document(textReader.Value, indexRoot, session);
doc.Persist(documentPlacement, session, true);
if (doc.DatabaseNumber != currentDocumentDatabaseNum)
{
session.FlushUpdates(session.OpenDatabase(currentDocumentDatabaseNum));
Console.WriteLine("Database: " + currentDocumentDatabaseNum +" is completed, done importing article " + docCount + " number of lines: " + textReader.LineNumber);
currentDocumentDatabaseNum = doc.DatabaseNumber;
}
//doc.Page.Database.Name = doc.Name;
}
else if (doc != null && pageText)
{
#if DEBUGx
Console.WriteLine(doc.Name + " line: " + textReader.LineNumber);
#endif
//if (textReader.LineNumber > 1000000)
//{
// session.Commit();
// return;
//}
DocumentText content = new DocumentText(textReader.Value, doc);
if (doc.DatabaseNumber != contentPlacement.TryDatabaseNumber)
contentPlacement = new Placement(doc.DatabaseNumber, (ushort)contentPlacement.StartPageNumber, 1, contentPlacement.MaxObjectsPerPage, contentPlacement.MaxPagesPerDatabase, false, false, 1, false);
content.Persist(contentPlacement, session, false);
Debug.Assert(content.DatabaseNumber == doc.DatabaseNumber);
doc.Content = content;
indexRoot.repository.documentSet.AddFast(doc);
if (++docCount % 1000000 == 0)
{
//session.Commit(false); // skip recovery check, we do it in BeginUpdate which is enough
Console.WriteLine("Done importing article " + docCount + " number of lines: " + textReader.LineNumber);
//session.BeginUpdate();
}
}
break;
case System.Xml.XmlNodeType.Whitespace:
xmlWhitespace = new XmlWhitespace(textReader.Value, xmlDocument);
break;
case System.Xml.XmlNodeType.XmlDeclaration:
break;
};
}
Console.WriteLine("Finished importing article " + docCount + " number of lines: " + textReader.LineNumber);
}
}
}
session.Commit();
}
Console.WriteLine(DateTime.Now.ToString() + ", done importing Wikipedia text");
}
public void MultipleThreadsAdding()
{
bool doClearAll = SessionBase.ClearAllCachedObjectsWhenDetectingUpdatedDatabase;
SessionBase.ClearAllCachedObjectsWhenDetectingUpdatedDatabase = false;
try
{
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
session.RegisterClass(typeof(AutoPlacement)); // build in type but not yet registered as a one
session.RegisterClass(typeof(ObservableList<int>));
session.RegisterClass(typeof(Dokument));
UInt32 dbNum = session.DatabaseNumberOf(typeof(Dokument));
Database db = session.OpenDatabase(dbNum, false, false);
if (db == null)
db = session.NewDatabase(dbNum, 0, typeof(Dokument).ToGenericTypeString());
Dokument doc = new Dokument();
session.Persist(doc);
session.Commit();
}
using (ServerClientSessionShared sharedReadSession = new ServerClientSessionShared(systemDir))
{
sharedReadSession.BeginRead();
Parallel.ForEach(Enumerable.Range(1, 3), (num) => LockConflict(sharedReadSession));
}
}
finally
{
SessionBase.ClearAllCachedObjectsWhenDetectingUpdatedDatabase = doClearAll;
}
}
static void Main(string[] args)
{
long triangles = 0;
try
{
using (SessionNoServer session = new SessionNoServer(systemDir))
{
int numberOfWorkerThreads = -1;
if (args.Length > 0)
{
if (!int.TryParse(args[0], out numberOfWorkerThreads))
{
Console.WriteLine("First parameter is numberOfWorkerThreads which must be an Int32");
}
}
bool useLinq = args.Length > 1;
Placement btreePlace = new Placement(40, 1, 1, 10000, 65500, true);
Placement edgeInfoPlace = new Placement(40, 10000, 1, 10000, 65500, true);
session.BeginUpdate();
BTreeMap <int, int[]> edges;
BTreeMapIterator <int, int[]> edgesItr;
int[] edge = null;
Database edgeDb = session.OpenDatabase(40, false, false);
if (edgeDb != null)
{
session.Commit();
session.BeginRead();
edges = (BTreeMap <int, int[]>)session.Open(40, 1, 1, false);
}
else
{
DatabaseLocation location = session.DatabaseLocations.Default();
//location.CompressPages = false; // no compression should make it faster (?)
session.NewDatabase(40, 395, "Edges");
edges = new BTreeMap <int, int[]>(null, session, 6000);
edges.Persist(btreePlace, session, true);
edgesItr = edges.Iterator();
using (StreamReader stream = new StreamReader(edgesInputFile, true))
{
int a;
int b;
string line;
string[] fields;
while ((line = stream.ReadLine()) != null)
{
fields = line.Split(' ');
if (!int.TryParse(fields[0], out a))
{
break;
}
b = int.Parse(fields[1]);
if (a != b)
{
if (edgesItr.CurrentKey() == a || edgesItr.GoTo(a))
{
edge = edgesItr.CurrentValue();
Array.Resize(ref edge, edge.Length + 1);
edge[edge.Length - 1] = b;
edgesItr.ReplaceValue(ref edge); // we need to update the value in the BTreeMap
}
else
{
edge = new int[1];
edge[0] = b;
edges.Add(a, edge);
}
}
}
}
edgesItr = edges.Iterator();
while (edgesItr.MoveNext())
{
edge = edgesItr.CurrentValue();
Array.Sort(edge);
edgesItr.ReplaceValue(ref edge);
}
session.Commit();
session.BeginRead();
}
Console.WriteLine("Number of Nodes found: " + edges.Count);
if (useLinq)
{
Console.WriteLine("Query using LINQ");
}
if (numberOfWorkerThreads > 0)
{
Console.WriteLine("Start of triangle discovery using " + numberOfWorkerThreads + " threads, time is " + DateTime.Now);
}
else if (numberOfWorkerThreads < 0)
{
Console.WriteLine("Start of triangle discovery using system automatically selected number of threads, time is " + DateTime.Now);
}
else
{
Console.WriteLine("Start of triangle discovery using main thread, time is " + DateTime.Now);
}
// Start counting triangles !
if (numberOfWorkerThreads != 0)
{
if (useLinq)
{ // Please help, I have not figured out how to properly do the triangle query using LINQ
int[] edge2values = null;
if (numberOfWorkerThreads > 0)
{
triangles = (from KeyValuePair <int, int[]> edgeFrom in edges
from int edgeTo1 in edgeFrom.Value
where edgeFrom.Key < edgeTo1
from int edgeTo2 in edgeFrom.Value
where edgeFrom.Key < edgeTo2 && edgeTo2 > edgeTo1 && edges.TryGetValue(edgeTo1, out edge2values) && Array.BinarySearch(edge2values, edgeTo2) >= 0
select edge).AsParallel().WithDegreeOfParallelism(numberOfWorkerThreads).Count();
}
else
{
triangles = (from KeyValuePair <int, int[]> edgeFrom in edges
from int edgeTo1 in edgeFrom.Value
where edgeFrom.Key < edgeTo1
from int edgeTo2 in edgeFrom.Value
where edgeFrom.Key <edgeTo2 && edgeTo2> edgeTo1 && edges.TryGetValue(edgeTo1, out edge2values) && Array.BinarySearch(edge2values, edgeTo2) >= 0
select edge).AsParallel().Count();
}
}
else
{
edgesItr = edges.Iterator();
ParallelOptions pOptions = new ParallelOptions();
pOptions.MaxDegreeOfParallelism = numberOfWorkerThreads;
// First type parameter is the type of the source elements
// Second type parameter is the type of the local data (subtotal)
Parallel.ForEach <KeyValuePair <int, int[]>, long>(edges, // source collection
pOptions,
() => 0, // method to initialize the local variable
(pair, loop, subtotal) => // method invoked by the loop on each iteration
{
int nodeId = pair.Key;
int[] nodeTo = pair.Value;
int stop = nodeTo.Length - 1;
int i = stop;
int edgeToStart, edgeTo;
int pos;
while (i >= 0)
{
int[] edgeInfo2;
edgeToStart = nodeTo[i--];
if (nodeId < edgeToStart)
{
if (edges.TryGetValue(edgeToStart, out edgeInfo2))
{
for (int j = stop; j >= i; j--)
{
edgeTo = nodeTo[j];
if (edgeToStart < edgeTo)
{
pos = Array.BinarySearch <int>(edgeInfo2, edgeTo);
if (pos >= 0)
{ // we know this one is connected to edgeInfo.From because it is part of edgeInfo.To
subtotal++;
}
}
else
{
break;
}
}
}
}
else
{
break;
}
}
return(subtotal);
},
// Method to be executed when all loops have completed.
// finalResult is the final value of subtotal. supplied by the ForEach method.
(finalResult) => Interlocked.Add(ref triangles, finalResult));
}
}
else if (useLinq)
{
triangles = queryUsingLINQ(edges);
}
else
{
triangles = discoverTrianglesSingleCore(edges);
}
session.Commit();
}
Console.WriteLine("Number of Triangles found: " + triangles);
}
catch (Exception e)
{
System.Console.WriteLine(e);
}
}
public MainWindow()
{
const ushort btreeNodeSize = 5000;
GCSettings.LatencyMode = GCLatencyMode.Batch;// try to keep the WeakIOptimizedPersistableReference objects around longer
dataGridList = new List <DataGrid>();
dataTableList = new List <DataTable>();
InitializeComponent();
session = new SessionNoServer(s_systemDir);
Placement placerIndexRoot = new Placement(IndexRoot.PlaceInDatabase);
session.BeginUpdate();
Console.WriteLine("Running with databases in directory: " + session.SystemDirectory);
File.Copy(s_licenseDbFile, Path.Combine(session.SystemDirectory, "4.odb"), true);
IndexRoot indexRoot;
Database db = session.OpenDatabase(IndexRoot.PlaceInDatabase, false, false);
if (db == null)
{
session.NewDatabase(IndexRoot.PlaceInDatabase, 0, "IndexRoot");
session.NewDatabase(Lexicon.PlaceInDatabase, 0, "Lexicon");
session.NewDatabase(Document.PlaceInDatabase, 0, "Document");
session.NewDatabase(Repository.PlaceInDatabase, 0, "Repository");
session.NewDatabase(DocumentText.PlaceInDatabase, 0, "DocumentText");
session.NewDatabase(Word.PlaceInDatabase, 0, "Word");
indexRoot = new IndexRoot(btreeNodeSize, session);
if (Directory.Exists(s_booksDir))
{
string[] directoryTextFiles = Directory.GetFiles(s_booksDir, "*.txt");
foreach (string fileName in directoryTextFiles)
{
listBoxPagesToAdd.Items.Add(fileName);
}
}
else
{
wordMinCt.Text = 1.ToString();
listBoxPagesToAdd.Items.Add("http://www.VelocityDB.com/");
// other database products
listBoxPagesToAdd.Items.Add("https://foundationdb.com/");
listBoxPagesToAdd.Items.Add("http://www.oracle.com/us/products/database/index.html");
listBoxPagesToAdd.Items.Add("http://www-01.ibm.com/software/data/db2/");
listBoxPagesToAdd.Items.Add("http://www.versant.com/");
listBoxPagesToAdd.Items.Add("http://web.progress.com/en/objectstore/");
listBoxPagesToAdd.Items.Add("https://www.mongodb.org/");
listBoxPagesToAdd.Items.Add("http://cassandra.apache.org/");
listBoxPagesToAdd.Items.Add("http://www.sybase.com/");
listBoxPagesToAdd.Items.Add("http://www.mcobject.com/perst");
listBoxPagesToAdd.Items.Add("http://www.marklogic.com/what-is-marklogic/");
listBoxPagesToAdd.Items.Add("http://hamsterdb.com/");
listBoxPagesToAdd.Items.Add("http://www.firebirdsql.org/");
listBoxPagesToAdd.Items.Add("http://www.h2database.com/");
listBoxPagesToAdd.Items.Add("http://www.oracle.com/technology/products/berkeley-db");
listBoxPagesToAdd.Items.Add("http://www.scimore.com/");
listBoxPagesToAdd.Items.Add("http://www.stsdb.com/");
listBoxPagesToAdd.Items.Add("http://www.sqlite.org/about.html");
listBoxPagesToAdd.Items.Add("http://www.mysql.com/products/enterprise/techspec.html");
listBoxPagesToAdd.Items.Add("http://www.objectivity.com");
listBoxPagesToAdd.Items.Add("http://vistadb.net/");
listBoxPagesToAdd.Items.Add("http://www.google.com/search?q=object+database&sourceid=ie7&rls=com.microsoft:en-us:IE-SearchBox&ie=&oe=");
}
indexRoot.Persist(session, indexRoot);
}
else
{
indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1));
}
if (indexRoot.repository.documentSet.Count > 0)
{
List <Document> docs = indexRoot.repository.documentSet.ToList <Document>().Take(50).ToList <Document>();
inDbListBox.ItemsSource = docs;
}
updateDataGrids(indexRoot);
session.Commit();
//verify();
}
static void Main(string[] args)
{
long triangles = 0;
try
{
using (SessionNoServer session = new SessionNoServer(systemDir))
{
int numberOfWorkerThreads = -1;
if (args.Length > 0)
{
if (!int.TryParse(args[0], out numberOfWorkerThreads))
Console.WriteLine("First parameter is numberOfWorkerThreads which must be an Int32");
}
bool useLinq = args.Length > 1;
Placement btreePlace = new Placement(40, 1, 1, 10000, 65500, true);
Placement edgeInfoPlace = new Placement(40, 10000, 1, 10000, 65500, true);
session.BeginUpdate();
BTreeMap<int, int[]> edges;
BTreeMapIterator<int, int[]> edgesItr;
int[] edge = null;
Database edgeDb = session.OpenDatabase(40, false, false);
if (edgeDb != null)
{
session.Commit();
session.BeginRead();
edges = (BTreeMap<int, int[]>)session.Open(40, 1, 1, false);
}
else
{
DatabaseLocation location = session.DatabaseLocations.Default();
//location.CompressPages = false; // no compression should make it faster (?)
session.NewDatabase(40, 395, "Edges");
edges = new BTreeMap<int, int[]>(null, session, 6000);
edges.Persist(btreePlace, session, true);
edgesItr = edges.Iterator();
using (StreamReader stream = new StreamReader(edgesInputFile, true))
{
int a;
int b;
string line;
string[] fields;
while ((line = stream.ReadLine()) != null)
{
fields = line.Split(' ');
if (!int.TryParse(fields[0], out a))
break;
b = int.Parse(fields[1]);
if (a != b)
{
if (edgesItr.CurrentKey() == a || edgesItr.GoTo(a))
{
edge = edgesItr.CurrentValue();
Array.Resize(ref edge, edge.Length + 1);
edge[edge.Length - 1] = b;
edgesItr.ReplaceValue(ref edge); // we need to update the value in the BTreeMap
}
else
{
edge = new int[1];
edge[0] = b;
edges.Add(a, edge);
}
}
}
}
edgesItr = edges.Iterator();
while (edgesItr.MoveNext())
{
edge = edgesItr.CurrentValue();
Array.Sort(edge);
edgesItr.ReplaceValue(ref edge);
}
session.Commit();
session.BeginRead();
}
Console.WriteLine("Number of Nodes found: " + edges.Count);
if (useLinq)
Console.WriteLine("Query using LINQ");
if (numberOfWorkerThreads > 0)
Console.WriteLine("Start of triangle discovery using " + numberOfWorkerThreads + " threads, time is " + DateTime.Now);
else if (numberOfWorkerThreads < 0)
Console.WriteLine("Start of triangle discovery using system automatically selected number of threads, time is " + DateTime.Now);
else
Console.WriteLine("Start of triangle discovery using main thread, time is " + DateTime.Now);
// Start counting triangles !
if (numberOfWorkerThreads != 0)
{
if (useLinq)
{ // Please help, I have not figured out how to properly do the triangle query using LINQ
int[] edge2values = null;
if (numberOfWorkerThreads > 0)
triangles = (from KeyValuePair<int, int[]> edgeFrom in edges
from int edgeTo1 in edgeFrom.Value
where edgeFrom.Key < edgeTo1
from int edgeTo2 in edgeFrom.Value
where edgeFrom.Key < edgeTo2 && edgeTo2 > edgeTo1 && edges.TryGetValue(edgeTo1, out edge2values) && Array.BinarySearch(edge2values, edgeTo2) >= 0
select edge).AsParallel().WithDegreeOfParallelism(numberOfWorkerThreads).Count();
else
triangles = (from KeyValuePair<int, int[]> edgeFrom in edges
from int edgeTo1 in edgeFrom.Value
where edgeFrom.Key < edgeTo1
from int edgeTo2 in edgeFrom.Value
where edgeFrom.Key < edgeTo2 && edgeTo2 > edgeTo1 && edges.TryGetValue(edgeTo1, out edge2values) && Array.BinarySearch(edge2values, edgeTo2) >= 0
select edge).AsParallel().Count();
}
else
{
edgesItr = edges.Iterator();
ParallelOptions pOptions = new ParallelOptions();
pOptions.MaxDegreeOfParallelism = numberOfWorkerThreads;
// First type parameter is the type of the source elements
// Second type parameter is the type of the local data (subtotal)
Parallel.ForEach<KeyValuePair<int, int[]>, long>(edges, // source collection
pOptions,
() => 0, // method to initialize the local variable
(pair, loop, subtotal) => // method invoked by the loop on each iteration
{
int nodeId = pair.Key;
int[] nodeTo = pair.Value;
int stop = nodeTo.Length - 1;
int i = stop;
int edgeToStart, edgeTo;
int pos;
while (i >= 0)
{
int[] edgeInfo2;
edgeToStart = nodeTo[i--];
if (nodeId < edgeToStart)
{
if (edges.TryGetValue(edgeToStart, out edgeInfo2))
{
for (int j = stop; j >= i; j--)
{
edgeTo = nodeTo[j];
if (edgeToStart < edgeTo)
{
pos = Array.BinarySearch<int>(edgeInfo2, edgeTo);
if (pos >= 0)
{ // we know this one is connected to edgeInfo.From because it is part of edgeInfo.To
subtotal++;
}
}
else
break;
}
}
}
else
break;
}
return subtotal;
},
// Method to be executed when all loops have completed.
// finalResult is the final value of subtotal. supplied by the ForEach method.
(finalResult) => Interlocked.Add(ref triangles, finalResult));
}
}
else if (useLinq)
triangles = queryUsingLINQ(edges);
else
triangles = discoverTrianglesSingleCore(edges);
session.Commit();
}
Console.WriteLine("Number of Triangles found: " + triangles);
}
catch (Exception e)
{
System.Console.WriteLine(e);
}
}
