public void aaaFakeLicenseDatabase() { Assert.Throws <NoValidVelocityDBLicenseFoundException>(() => { using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); Database database; License license = new License("Mats", 1, null, null, null, 99999, DateTime.MaxValue, 9999, 99, 9999); Placement placer = new Placement(License.PlaceInDatabase, 1, 1, 1); license.Persist(placer, session); for (uint i = 10; i < 20; i++) { database = session.NewDatabase(i); Assert.NotNull(database); } session.Commit(); File.Copy(Path.Combine(systemDir, "20.odb"), Path.Combine(systemDir, "4.odb")); session.BeginUpdate(); for (uint i = 21; i < 30; i++) { database = session.NewDatabase(i); Assert.NotNull(database); } session.RegisterClass(typeof(BTreeSet <int>)); session.RegisterClass(typeof(BTreeSet <long>)); session.RegisterClass(typeof(BTreeSet <DateTime>)); session.RegisterClass(typeof(BTreeSet <double>)); session.RegisterClass(typeof(BTreeMap <string, double>)); session.Commit(); } }); }
public void aaaFakeLicenseDatabase() { Assert.Throws<NoValidVelocityDBLicenseFoundException>(() => { using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); Database database; License license = new License("Mats", 1, null, null, null, 99999, DateTime.MaxValue, 9999, 99, 9999); Placement placer = new Placement(License.PlaceInDatabase, 1, 1, 1); license.Persist(placer, session); for (uint i = 10; i < 20; i++) { database = session.NewDatabase(i); Assert.NotNull(database); } session.Commit(); File.Copy(Path.Combine(systemDir, "20.odb"), Path.Combine(systemDir, "4.odb")); session.BeginUpdate(); for (uint i = 21; i < 30; i++) { database = session.NewDatabase(i); Assert.NotNull(database); } session.RegisterClass(typeof(VelocityDbSchema.Samples.Sample1.Person)); Graph g = new Graph(session); session.Persist(g); session.Commit(); } }); }
public void aaaFakeLicenseDatabase() { using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); Database database; License license = new License("Mats", 1, null, null, null, 99999, DateTime.MaxValue, 9999, 99, 9999); Placement placer = new Placement(License.PlaceInDatabase, 1, 1, 1); license.Persist(placer, session); for (uint i = 10; i < 20; i++) { database = session.NewDatabase(i); Assert.NotNull(database); } session.Commit(); File.Copy(Path.Combine(systemDir, "20.odb"), Path.Combine(systemDir, "4.odb")); session.BeginUpdate(); for (uint i = 21; i < 30; i++) { database = session.NewDatabase(i); Assert.NotNull(database); } session.RegisterClass(typeof(VelocityDbSchema.Samples.Sample1.Person)); Graph g = new Graph(session); session.Persist(g); session.Commit(); } }
public void License() { using (SessionNoServer session = new SessionNoServer(systemDir)) { Database database; session.BeginUpdate(); for (uint i = 60000000; i < 60000010; i++) { database = session.NewDatabase(i); Assert.NotNull(database); } session.Commit(); session.BeginUpdate(); for (uint i = 60000000; i < 60000010; i++) { database = session.OpenDatabase(i); Assert.NotNull(database); } session.Commit(); session.BeginUpdate(); for (uint i = 60000000; i < 60000010; i++) { database = session.OpenDatabase(i); session.DeleteDatabase(database); } session.Commit(); } }
public void TwoUpdaters1() { Assert.Throws<OptimisticLockingFailed>(() => { UInt64 id; using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); Man man = new Man(); man.Persist(session, man); id = man.Id; session.Commit(); session.BeginUpdate(); man.Age = ++man.Age; Database db = session.NewDatabase(3567); using (SessionNoServer session2 = new SessionNoServer(systemDir)) { session2.BeginUpdate(); Man man2 = (Man)session2.Open(id); Assert.Less(man2.Age, man.Age); man2.Age = ++man.Age; session2.Commit(); } session.DeleteDatabase(db); session.Commit(); // OptimisticLockingFailed here } }); }
public void aCreateDatabases() { Database database; using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); for (uint i = 50000000; i < 50001000; i++) { database = session.NewDatabase(i); Assert.NotNull(database); } session.Commit(); } using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); for (uint i = 50000000; i < 50001000; i++) { database = session.OpenDatabase(i); Assert.NotNull(database); } session.Commit(); } }
public void CreateLocationSameHost() { using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); DatabaseLocation remoteLocation = new DatabaseLocation(Dns.GetHostName(), location2Dir, locationStartDbNum, locationEndDbNum, session, PageInfo.compressionKind.LZ4, 0); remoteLocation = session.NewLocation(remoteLocation); Database database = session.NewDatabase(remoteLocation.StartDatabaseNumber); Assert.NotNull(database); session.Commit(); } using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); Database database = session.OpenDatabase(locationStartDbNum, false); Assert.NotNull(database); session.DeleteDatabase(database); session.Commit(); } using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); foreach (DatabaseLocation loc in session.DatabaseLocations) { Console.WriteLine(loc.ToStringDetails(session, false)); } session.DeleteLocation(session.DatabaseLocations.LocationForDb(locationStartDbNum)); foreach (DatabaseLocation loc in session.DatabaseLocations) { Console.WriteLine(loc.ToStringDetails(session, false)); } session.Commit(); } }
public void TwoUpdaters1() { Assert.Throws <OptimisticLockingFailed>(() => { UInt64 id; using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); Man man = new Man(); man.Persist(session, man); id = man.Id; session.Commit(); session.BeginUpdate(); man.Age = ++man.Age; Database db = session.NewDatabase(3567); using (SessionNoServer session2 = new SessionNoServer(systemDir)) { session2.BeginUpdate(); Man man2 = (Man)session2.Open(id); Assert.Less(man2.Age, man.Age); man2.Age = ++man.Age; session2.Commit(); } session.DeleteDatabase(db); session.Commit(); // OptimisticLockingFailed here session.Verify(); } }); }
public void MultipleThreadsAdding() { bool doClearAll = SessionBase.ClearAllCachedObjectsWhenDetectingUpdatedDatabase; SessionBase.ClearAllCachedObjectsWhenDetectingUpdatedDatabase = false; try { using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); session.RegisterClass(typeof(AutoPlacement)); // build in type but not yet registered as a one session.RegisterClass(typeof(ObservableList <int>)); session.RegisterClass(typeof(Dokument)); UInt32 dbNum = session.DatabaseNumberOf(typeof(Dokument)); Database db = session.OpenDatabase(dbNum, false, false); if (db == null) { db = session.NewDatabase(dbNum, 0, typeof(Dokument).ToGenericTypeString()); } Dokument doc = new Dokument(); session.Persist(doc); session.Commit(); } using (ServerClientSessionShared sharedReadSession = new ServerClientSessionShared(systemDir)) { sharedReadSession.BeginRead(); Parallel.ForEach(Enumerable.Range(1, 3), (num) => LockConflict(sharedReadSession)); } } finally { SessionBase.ClearAllCachedObjectsWhenDetectingUpdatedDatabase = doClearAll; } }
public void CreateLocationSameHost() { using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); DatabaseLocation remoteLocation = new DatabaseLocation(Dns.GetHostName(), location2Dir, locationStartDbNum, locationEndDbNum, session, PageInfo.compressionKind.LZ4, 0); remoteLocation = session.NewLocation(remoteLocation); Database database = session.NewDatabase(remoteLocation.StartDatabaseNumber); Assert.NotNull(database); session.Commit(); } using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); Database database = session.OpenDatabase(locationStartDbNum, false); Assert.NotNull(database); session.DeleteDatabase(database); session.Commit(); } using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); foreach (DatabaseLocation loc in session.DatabaseLocations) Console.WriteLine(loc.ToStringDetails(session, false)); session.DeleteLocation(session.DatabaseLocations.LocationForDb(locationStartDbNum)); foreach (DatabaseLocation loc in session.DatabaseLocations) Console.WriteLine(loc.ToStringDetails(session, false)); session.Commit(); } }
public void aCreateDatabases() { Database database; using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); for (uint i = 50000000; i < 50001000; i++) { database = session.NewDatabase(i); Assert.NotNull(database); } session.Commit(); } using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); for (uint i = 50000000; i < 50001000; i++) { database = session.OpenDatabase(i); Assert.NotNull(database); } session.Commit(); } }
public void aSyncNewDatabases() { using (SessionBase session = new SessionNoServer(s_sync1)) { session.EnableSyncByTrackingChanges = true; using (var trans = session.BeginUpdate()) { for (uint i = 10; i < 50; i++) { var database = session.NewDatabase(i); Assert.NotNull(database); } session.Commit(); } } using (SessionBase readFromSession = new SessionNoServer(s_sync1)) { using (SessionBase updateSession = new SessionNoServer(s_sync2)) { updateSession.SyncWith(readFromSession); } } using (SessionBase readFromSession = new SessionNoServer(s_sync1)) { readFromSession.BeginRead(); using (SessionBase updateSession = new SessionNoServer(s_sync2)) { using (var trans = updateSession.BeginRead()) { Assert.AreEqual(updateSession.OpenAllDatabases().Count, readFromSession.OpenAllDatabases().Count - 1); // - 1 due to additional change tracking databases in original } } } }
public void aSyncNewDatabases() { using (SessionBase session = new SessionNoServer(s_sync1)) { session.EnableSyncByTrackingChanges = true; using (var trans = session.BeginUpdate()) { for (uint i = 10; i < 50; i++) { var database = session.NewDatabase(i); Assert.NotNull(database); } session.Commit(); } } using (SessionBase readFromSession = new SessionNoServer(s_sync1)) { using (SessionBase updateSession = new SessionNoServer(s_sync2)) { updateSession.SyncWith(readFromSession); } } using (SessionBase readFromSession = new SessionNoServer(s_sync1)) { readFromSession.BeginRead(); using (SessionBase updateSession = new SessionNoServer(s_sync2)) { using (var trans = updateSession.BeginRead()) { Assert.AreEqual(updateSession.OpenAllDatabases().Count, readFromSession.OpenAllDatabases().Count - 1); // - 1 due to additional change tracking databases in original } } } }
static void importEntireWikipedia() { const ushort btreeNodeSize = 10000; Console.WriteLine(DateTime.Now.ToString() + ", start importing Wikipedia text"); //System.Xml.Schema.XmlSchema docSchema; //using (System.Xml.XmlTextReader schemaReader = new System.Xml.XmlTextReader("c:\\export-0_5.xsd")) //{ // docSchema = System.Xml.Schema.XmlSchema.Read(schemaReader, ValidationCallBack); // } int docCount = 0; using (SessionNoServer session = new SessionNoServer(s_systemDir, 5000, false, false, CacheEnum.No)) // turn of page and object caching { Console.WriteLine("Running with databases in directory: " + session.SystemDirectory); //GCSettings.LatencyMode = GCLatencyMode.Batch;// try to keep the WeakIOptimizedPersistableReference objects around longer Placement documentPlacement = new Placement(Document.PlaceInDatabase, 1003, 1, 500, 1000, false, false, 1000, false); Placement contentPlacement = new Placement(Document.PlaceInDatabase, 1, 1, 500, UInt16.MaxValue, false, false, 1, false); XmlComment xmlComment; XmlElement xmlElement; XmlEntity xmlEntity; XmlText xmlText; XmlWhitespace xmlWhitespace; session.BeginUpdate(); File.Copy(s_licenseDbFile, System.IO.Path.Combine(session.SystemDirectory, "4.odb"), true); // register all database schema classes used by the application in advance to avoid lock conflict later in parallell indexing session.RegisterClass(typeof(Repository)); session.RegisterClass(typeof(IndexRoot)); session.RegisterClass(typeof(Document)); session.RegisterClass(typeof(Lexicon)); session.RegisterClass(typeof(DocumentText)); session.RegisterClass(typeof(Word)); session.RegisterClass(typeof(WordGlobal)); session.RegisterClass(typeof(WordHit)); session.RegisterClass(typeof(BTreeSet <Document>)); session.RegisterClass(typeof(OidShort)); session.RegisterClass(typeof(BTreeMap <Word, WordHit>)); session.RegisterClass(typeof(HashCodeComparer <Word>)); session.RegisterClass(typeof(BTreeSetOidShort <Word>)); session.RegisterClass(typeof(BTreeMapOidShort <Word, WordHit>)); Database db = session.OpenDatabase(IndexRoot.PlaceInDatabase, false, false); if (db != null) { outputSomeInfo(session); session.Abort(); return; } session.NewDatabase(IndexRoot.PlaceInDatabase, 0, "IndexRoot"); session.NewDatabase(Lexicon.PlaceInDatabase, 0, "Lexicon"); session.NewDatabase(Repository.PlaceInDatabase, 0, "Repository"); for (UInt32 i = 40; i <= 186; i++) { session.NewDatabase(i, 512, "Document"); // pre allocate 146 Document databases presized to 512MB each } //session.SetTraceDbActivity(Lexicon.PlaceInDatabase); //session.SetTraceAllDbActivity(); XmlDocument xmlDocument = new XmlDocument("enwiki-latest-pages-articles.xml"); IndexRoot indexRoot = new IndexRoot(btreeNodeSize, session); indexRoot.Persist(session, indexRoot, true); Document doc = null; bool titleElement = false; bool pageText = false; UInt32 currentDocumentDatabaseNum = documentPlacement.StartDatabaseNumber; using (FileStream fs = new FileStream(s_wikipediaXmlFile, FileMode.Open)) { //using (GZipStream zipStream = new GZipStream(fs, CompressionMode.Decompress)) // if input was a .gz file { using (System.Xml.XmlTextReader textReader = new System.Xml.XmlTextReader(fs)) { while (textReader.Read()) { System.Xml.XmlNodeType nodeType = textReader.NodeType; switch (nodeType) { case System.Xml.XmlNodeType.Attribute: break; case System.Xml.XmlNodeType.CDATA: break; case System.Xml.XmlNodeType.Comment: xmlComment = new XmlComment(textReader.Value, xmlDocument); break; case System.Xml.XmlNodeType.Document: break; case System.Xml.XmlNodeType.DocumentFragment: break; case System.Xml.XmlNodeType.DocumentType: break; case System.Xml.XmlNodeType.Element: xmlElement = new XmlElement(textReader.Prefix, textReader.LocalName, textReader.NamespaceURI, xmlDocument); if (textReader.LocalName == "title") { titleElement = true; } else if (textReader.LocalName == "text") { pageText = true; } break; case System.Xml.XmlNodeType.EndElement: if (textReader.LocalName == "title" && doc != null) { titleElement = false; } else if (textReader.LocalName == "text" && doc != null) { pageText = false; } break; case System.Xml.XmlNodeType.EndEntity: break; case System.Xml.XmlNodeType.Entity: xmlEntity = new XmlEntity(textReader.LocalName, xmlDocument); break; case System.Xml.XmlNodeType.EntityReference: break; case System.Xml.XmlNodeType.None: break; case System.Xml.XmlNodeType.Notation: break; case System.Xml.XmlNodeType.ProcessingInstruction: break; case System.Xml.XmlNodeType.SignificantWhitespace: break; case System.Xml.XmlNodeType.Text: xmlText = new XmlText(textReader.Value, xmlDocument); if (titleElement) { doc = new Document(textReader.Value, indexRoot, session); doc.Persist(documentPlacement, session, true); if (doc.DatabaseNumber != currentDocumentDatabaseNum) { session.FlushUpdates(session.OpenDatabase(currentDocumentDatabaseNum)); Console.WriteLine("Database: " + currentDocumentDatabaseNum + " is completed, done importing article " + docCount + " number of lines: " + textReader.LineNumber); currentDocumentDatabaseNum = doc.DatabaseNumber; } //doc.Page.Database.Name = doc.Name; } else if (doc != null && pageText) { #if DEBUGx Console.WriteLine(doc.Name + " line: " + textReader.LineNumber); #endif //if (textReader.LineNumber > 1000000) //{ // session.Commit(); // return; //} DocumentText content = new DocumentText(textReader.Value, doc); if (doc.DatabaseNumber != contentPlacement.TryDatabaseNumber) { contentPlacement = new Placement(doc.DatabaseNumber, (ushort)contentPlacement.StartPageNumber, 1, contentPlacement.MaxObjectsPerPage, contentPlacement.MaxPagesPerDatabase, false, false, 1, false); } content.Persist(contentPlacement, session, false); Debug.Assert(content.DatabaseNumber == doc.DatabaseNumber); doc.Content = content; indexRoot.repository.documentSet.AddFast(doc); if (++docCount % 1000000 == 0) { //session.Commit(false); // skip recovery check, we do it in BeginUpdate which is enough Console.WriteLine("Done importing article " + docCount + " number of lines: " + textReader.LineNumber); //session.BeginUpdate(); } } break; case System.Xml.XmlNodeType.Whitespace: xmlWhitespace = new XmlWhitespace(textReader.Value, xmlDocument); break; case System.Xml.XmlNodeType.XmlDeclaration: break; } ; } Console.WriteLine("Finished importing article " + docCount + " number of lines: " + textReader.LineNumber); } } } session.Commit(); } Console.WriteLine(DateTime.Now.ToString() + ", done importing Wikipedia text"); }
public MainWindow() { const ushort btreeNodeSize = 5000; GCSettings.LatencyMode = GCLatencyMode.Batch;// try to keep the WeakIOptimizedPersistableReference objects around longer dataGridList = new List<DataGrid>(); dataTableList = new List<DataTable>(); InitializeComponent(); session = new SessionNoServer(s_systemDir); Placement placerIndexRoot = new Placement(IndexRoot.PlaceInDatabase); session.BeginUpdate(); Console.WriteLine("Running with databases in directory: " + session.SystemDirectory); File.Copy(s_licenseDbFile, Path.Combine(session.SystemDirectory, "4.odb"), true); IndexRoot indexRoot; Database db = session.OpenDatabase(IndexRoot.PlaceInDatabase, false, false); if (db == null) { session.NewDatabase(IndexRoot.PlaceInDatabase, 0, "IndexRoot"); session.NewDatabase(Lexicon.PlaceInDatabase, 0, "Lexicon"); session.NewDatabase(Document.PlaceInDatabase, 0, "Document"); session.NewDatabase(Repository.PlaceInDatabase, 0, "Repository"); session.NewDatabase(DocumentText.PlaceInDatabase, 0, "DocumentText"); session.NewDatabase(Word.PlaceInDatabase, 0, "Word"); indexRoot = new IndexRoot(btreeNodeSize, session); if (Directory.Exists(s_booksDir)) { string[] directoryTextFiles = Directory.GetFiles(s_booksDir, "*.txt"); foreach (string fileName in directoryTextFiles) { listBoxPagesToAdd.Items.Add(fileName); } } else { wordMinCt.Text = 1.ToString(); listBoxPagesToAdd.Items.Add("http://www.VelocityDB.com/"); // other database products listBoxPagesToAdd.Items.Add("https://foundationdb.com/"); listBoxPagesToAdd.Items.Add("http://www.oracle.com/us/products/database/index.html"); listBoxPagesToAdd.Items.Add("http://www-01.ibm.com/software/data/db2/"); listBoxPagesToAdd.Items.Add("http://www.versant.com/"); listBoxPagesToAdd.Items.Add("http://web.progress.com/en/objectstore/"); listBoxPagesToAdd.Items.Add("https://www.mongodb.org/"); listBoxPagesToAdd.Items.Add("http://cassandra.apache.org/"); listBoxPagesToAdd.Items.Add("http://www.sybase.com/"); listBoxPagesToAdd.Items.Add("http://www.mcobject.com/perst"); listBoxPagesToAdd.Items.Add("http://www.marklogic.com/what-is-marklogic/"); listBoxPagesToAdd.Items.Add("http://hamsterdb.com/"); listBoxPagesToAdd.Items.Add("http://www.firebirdsql.org/"); listBoxPagesToAdd.Items.Add("http://www.h2database.com/"); listBoxPagesToAdd.Items.Add("http://www.oracle.com/technology/products/berkeley-db"); listBoxPagesToAdd.Items.Add("http://www.scimore.com/"); listBoxPagesToAdd.Items.Add("http://www.stsdb.com/"); listBoxPagesToAdd.Items.Add("http://www.sqlite.org/about.html"); listBoxPagesToAdd.Items.Add("http://www.mysql.com/products/enterprise/techspec.html"); listBoxPagesToAdd.Items.Add("http://www.objectivity.com"); listBoxPagesToAdd.Items.Add("http://vistadb.net/"); listBoxPagesToAdd.Items.Add("http://www.google.com/search?q=object+database&sourceid=ie7&rls=com.microsoft:en-us:IE-SearchBox&ie=&oe="); } indexRoot.Persist(session, indexRoot); } else indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1)); if (indexRoot.repository.documentSet.Count > 0) { List<Document> docs = indexRoot.repository.documentSet.ToList<Document>().Take(50).ToList<Document>(); inDbListBox.ItemsSource = docs; } updateDataGrids(indexRoot); session.Commit(); //verify(); }
static void importEntireWikipedia() { const ushort btreeNodeSize = 10000; Console.WriteLine(DateTime.Now.ToString() + ", start importing Wikipedia text"); //System.Xml.Schema.XmlSchema docSchema; //using (System.Xml.XmlTextReader schemaReader = new System.Xml.XmlTextReader("c:\\export-0_5.xsd")) //{ // docSchema = System.Xml.Schema.XmlSchema.Read(schemaReader, ValidationCallBack); // } int docCount = 0; using (SessionNoServer session = new SessionNoServer(s_systemDir, 5000, false, false, CacheEnum.No)) // turn of page and object caching { Console.WriteLine("Running with databases in directory: " + session.SystemDirectory); //GCSettings.LatencyMode = GCLatencyMode.Batch;// try to keep the WeakIOptimizedPersistableReference objects around longer Placement documentPlacement = new Placement(Document.PlaceInDatabase, 1003, 1, 500, 1000, false, false, 1000, false); Placement contentPlacement = new Placement(Document.PlaceInDatabase, 1, 1, 500, UInt16.MaxValue, false, false, 1, false); XmlComment xmlComment; XmlElement xmlElement; XmlEntity xmlEntity; XmlText xmlText; XmlWhitespace xmlWhitespace; session.BeginUpdate(); File.Copy(s_licenseDbFile, System.IO.Path.Combine(session.SystemDirectory, "4.odb"), true); // register all database schema classes used by the application in advance to avoid lock conflict later in parallell indexing session.RegisterClass(typeof(Repository)); session.RegisterClass(typeof(IndexRoot)); session.RegisterClass(typeof(Document)); session.RegisterClass(typeof(Lexicon)); session.RegisterClass(typeof(DocumentText)); session.RegisterClass(typeof(Word)); session.RegisterClass(typeof(WordGlobal)); session.RegisterClass(typeof(WordHit)); session.RegisterClass(typeof(BTreeSet<Document>)); session.RegisterClass(typeof(OidShort)); session.RegisterClass(typeof(BTreeMap<Word, WordHit>)); session.RegisterClass(typeof(HashCodeComparer<Word>)); session.RegisterClass(typeof(BTreeSetOidShort<Word>)); session.RegisterClass(typeof(BTreeMapOidShort<Word, WordHit>)); Database db = session.OpenDatabase(IndexRoot.PlaceInDatabase, false, false); if (db != null) { outputSomeInfo(session); session.Abort(); return; } session.NewDatabase(IndexRoot.PlaceInDatabase, 0, "IndexRoot"); session.NewDatabase(Lexicon.PlaceInDatabase, 0, "Lexicon"); session.NewDatabase(Repository.PlaceInDatabase, 0, "Repository"); for (UInt32 i = 40; i <= 186; i++) { session.NewDatabase(i, 512, "Document"); // pre allocate 146 Document databases presized to 512MB each } //session.SetTraceDbActivity(Lexicon.PlaceInDatabase); //session.SetTraceAllDbActivity(); XmlDocument xmlDocument = new XmlDocument("enwiki-latest-pages-articles.xml"); IndexRoot indexRoot = new IndexRoot(btreeNodeSize, session); indexRoot.Persist(session, indexRoot, true); Document doc = null; bool titleElement = false; bool pageText = false; UInt32 currentDocumentDatabaseNum = documentPlacement.StartDatabaseNumber; using (FileStream fs = new FileStream(s_wikipediaXmlFile, FileMode.Open)) { //using (GZipStream zipStream = new GZipStream(fs, CompressionMode.Decompress)) // if input was a .gz file { using (System.Xml.XmlTextReader textReader = new System.Xml.XmlTextReader(fs)) { while (textReader.Read()) { System.Xml.XmlNodeType nodeType = textReader.NodeType; switch (nodeType) { case System.Xml.XmlNodeType.Attribute: break; case System.Xml.XmlNodeType.CDATA: break; case System.Xml.XmlNodeType.Comment: xmlComment = new XmlComment(textReader.Value, xmlDocument); break; case System.Xml.XmlNodeType.Document: break; case System.Xml.XmlNodeType.DocumentFragment: break; case System.Xml.XmlNodeType.DocumentType: break; case System.Xml.XmlNodeType.Element: xmlElement = new XmlElement(textReader.Prefix, textReader.LocalName, textReader.NamespaceURI, xmlDocument); if (textReader.LocalName == "title") titleElement = true; else if (textReader.LocalName == "text") pageText = true; break; case System.Xml.XmlNodeType.EndElement: if (textReader.LocalName == "title" && doc != null) titleElement = false; else if (textReader.LocalName == "text" && doc != null) pageText = false; break; case System.Xml.XmlNodeType.EndEntity: break; case System.Xml.XmlNodeType.Entity: xmlEntity = new XmlEntity(textReader.LocalName, xmlDocument); break; case System.Xml.XmlNodeType.EntityReference: break; case System.Xml.XmlNodeType.None: break; case System.Xml.XmlNodeType.Notation: break; case System.Xml.XmlNodeType.ProcessingInstruction: break; case System.Xml.XmlNodeType.SignificantWhitespace: break; case System.Xml.XmlNodeType.Text: xmlText = new XmlText(textReader.Value, xmlDocument); if (titleElement) { doc = new Document(textReader.Value, indexRoot, session); doc.Persist(documentPlacement, session, true); if (doc.DatabaseNumber != currentDocumentDatabaseNum) { session.FlushUpdates(session.OpenDatabase(currentDocumentDatabaseNum)); Console.WriteLine("Database: " + currentDocumentDatabaseNum +" is completed, done importing article " + docCount + " number of lines: " + textReader.LineNumber); currentDocumentDatabaseNum = doc.DatabaseNumber; } //doc.Page.Database.Name = doc.Name; } else if (doc != null && pageText) { #if DEBUGx Console.WriteLine(doc.Name + " line: " + textReader.LineNumber); #endif //if (textReader.LineNumber > 1000000) //{ // session.Commit(); // return; //} DocumentText content = new DocumentText(textReader.Value, doc); if (doc.DatabaseNumber != contentPlacement.TryDatabaseNumber) contentPlacement = new Placement(doc.DatabaseNumber, (ushort)contentPlacement.StartPageNumber, 1, contentPlacement.MaxObjectsPerPage, contentPlacement.MaxPagesPerDatabase, false, false, 1, false); content.Persist(contentPlacement, session, false); Debug.Assert(content.DatabaseNumber == doc.DatabaseNumber); doc.Content = content; indexRoot.repository.documentSet.AddFast(doc); if (++docCount % 1000000 == 0) { //session.Commit(false); // skip recovery check, we do it in BeginUpdate which is enough Console.WriteLine("Done importing article " + docCount + " number of lines: " + textReader.LineNumber); //session.BeginUpdate(); } } break; case System.Xml.XmlNodeType.Whitespace: xmlWhitespace = new XmlWhitespace(textReader.Value, xmlDocument); break; case System.Xml.XmlNodeType.XmlDeclaration: break; }; } Console.WriteLine("Finished importing article " + docCount + " number of lines: " + textReader.LineNumber); } } } session.Commit(); } Console.WriteLine(DateTime.Now.ToString() + ", done importing Wikipedia text"); }
public void MultipleThreadsAdding() { bool doClearAll = SessionBase.ClearAllCachedObjectsWhenDetectingUpdatedDatabase; SessionBase.ClearAllCachedObjectsWhenDetectingUpdatedDatabase = false; try { using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); session.RegisterClass(typeof(AutoPlacement)); // build in type but not yet registered as a one session.RegisterClass(typeof(ObservableList<int>)); session.RegisterClass(typeof(Dokument)); UInt32 dbNum = session.DatabaseNumberOf(typeof(Dokument)); Database db = session.OpenDatabase(dbNum, false, false); if (db == null) db = session.NewDatabase(dbNum, 0, typeof(Dokument).ToGenericTypeString()); Dokument doc = new Dokument(); session.Persist(doc); session.Commit(); } using (ServerClientSessionShared sharedReadSession = new ServerClientSessionShared(systemDir)) { sharedReadSession.BeginRead(); Parallel.ForEach(Enumerable.Range(1, 3), (num) => LockConflict(sharedReadSession)); } } finally { SessionBase.ClearAllCachedObjectsWhenDetectingUpdatedDatabase = doClearAll; } }
static void Main(string[] args) { long triangles = 0; try { using (SessionNoServer session = new SessionNoServer(systemDir)) { int numberOfWorkerThreads = -1; if (args.Length > 0) { if (!int.TryParse(args[0], out numberOfWorkerThreads)) { Console.WriteLine("First parameter is numberOfWorkerThreads which must be an Int32"); } } bool useLinq = args.Length > 1; Placement btreePlace = new Placement(40, 1, 1, 10000, 65500, true); Placement edgeInfoPlace = new Placement(40, 10000, 1, 10000, 65500, true); session.BeginUpdate(); BTreeMap <int, int[]> edges; BTreeMapIterator <int, int[]> edgesItr; int[] edge = null; Database edgeDb = session.OpenDatabase(40, false, false); if (edgeDb != null) { session.Commit(); session.BeginRead(); edges = (BTreeMap <int, int[]>)session.Open(40, 1, 1, false); } else { DatabaseLocation location = session.DatabaseLocations.Default(); //location.CompressPages = false; // no compression should make it faster (?) session.NewDatabase(40, 395, "Edges"); edges = new BTreeMap <int, int[]>(null, session, 6000); edges.Persist(btreePlace, session, true); edgesItr = edges.Iterator(); using (StreamReader stream = new StreamReader(edgesInputFile, true)) { int a; int b; string line; string[] fields; while ((line = stream.ReadLine()) != null) { fields = line.Split(' '); if (!int.TryParse(fields[0], out a)) { break; } b = int.Parse(fields[1]); if (a != b) { if (edgesItr.CurrentKey() == a || edgesItr.GoTo(a)) { edge = edgesItr.CurrentValue(); Array.Resize(ref edge, edge.Length + 1); edge[edge.Length - 1] = b; edgesItr.ReplaceValue(ref edge); // we need to update the value in the BTreeMap } else { edge = new int[1]; edge[0] = b; edges.Add(a, edge); } } } } edgesItr = edges.Iterator(); while (edgesItr.MoveNext()) { edge = edgesItr.CurrentValue(); Array.Sort(edge); edgesItr.ReplaceValue(ref edge); } session.Commit(); session.BeginRead(); } Console.WriteLine("Number of Nodes found: " + edges.Count); if (useLinq) { Console.WriteLine("Query using LINQ"); } if (numberOfWorkerThreads > 0) { Console.WriteLine("Start of triangle discovery using " + numberOfWorkerThreads + " threads, time is " + DateTime.Now); } else if (numberOfWorkerThreads < 0) { Console.WriteLine("Start of triangle discovery using system automatically selected number of threads, time is " + DateTime.Now); } else { Console.WriteLine("Start of triangle discovery using main thread, time is " + DateTime.Now); } // Start counting triangles ! if (numberOfWorkerThreads != 0) { if (useLinq) { // Please help, I have not figured out how to properly do the triangle query using LINQ int[] edge2values = null; if (numberOfWorkerThreads > 0) { triangles = (from KeyValuePair <int, int[]> edgeFrom in edges from int edgeTo1 in edgeFrom.Value where edgeFrom.Key < edgeTo1 from int edgeTo2 in edgeFrom.Value where edgeFrom.Key < edgeTo2 && edgeTo2 > edgeTo1 && edges.TryGetValue(edgeTo1, out edge2values) && Array.BinarySearch(edge2values, edgeTo2) >= 0 select edge).AsParallel().WithDegreeOfParallelism(numberOfWorkerThreads).Count(); } else { triangles = (from KeyValuePair <int, int[]> edgeFrom in edges from int edgeTo1 in edgeFrom.Value where edgeFrom.Key < edgeTo1 from int edgeTo2 in edgeFrom.Value where edgeFrom.Key <edgeTo2 && edgeTo2> edgeTo1 && edges.TryGetValue(edgeTo1, out edge2values) && Array.BinarySearch(edge2values, edgeTo2) >= 0 select edge).AsParallel().Count(); } } else { edgesItr = edges.Iterator(); ParallelOptions pOptions = new ParallelOptions(); pOptions.MaxDegreeOfParallelism = numberOfWorkerThreads; // First type parameter is the type of the source elements // Second type parameter is the type of the local data (subtotal) Parallel.ForEach <KeyValuePair <int, int[]>, long>(edges, // source collection pOptions, () => 0, // method to initialize the local variable (pair, loop, subtotal) => // method invoked by the loop on each iteration { int nodeId = pair.Key; int[] nodeTo = pair.Value; int stop = nodeTo.Length - 1; int i = stop; int edgeToStart, edgeTo; int pos; while (i >= 0) { int[] edgeInfo2; edgeToStart = nodeTo[i--]; if (nodeId < edgeToStart) { if (edges.TryGetValue(edgeToStart, out edgeInfo2)) { for (int j = stop; j >= i; j--) { edgeTo = nodeTo[j]; if (edgeToStart < edgeTo) { pos = Array.BinarySearch <int>(edgeInfo2, edgeTo); if (pos >= 0) { // we know this one is connected to edgeInfo.From because it is part of edgeInfo.To subtotal++; } } else { break; } } } } else { break; } } return(subtotal); }, // Method to be executed when all loops have completed. // finalResult is the final value of subtotal. supplied by the ForEach method. (finalResult) => Interlocked.Add(ref triangles, finalResult)); } } else if (useLinq) { triangles = queryUsingLINQ(edges); } else { triangles = discoverTrianglesSingleCore(edges); } session.Commit(); } Console.WriteLine("Number of Triangles found: " + triangles); } catch (Exception e) { System.Console.WriteLine(e); } }
public MainWindow() { const ushort btreeNodeSize = 5000; GCSettings.LatencyMode = GCLatencyMode.Batch;// try to keep the WeakIOptimizedPersistableReference objects around longer dataGridList = new List <DataGrid>(); dataTableList = new List <DataTable>(); InitializeComponent(); session = new SessionNoServer(s_systemDir); Placement placerIndexRoot = new Placement(IndexRoot.PlaceInDatabase); session.BeginUpdate(); Console.WriteLine("Running with databases in directory: " + session.SystemDirectory); File.Copy(s_licenseDbFile, Path.Combine(session.SystemDirectory, "4.odb"), true); IndexRoot indexRoot; Database db = session.OpenDatabase(IndexRoot.PlaceInDatabase, false, false); if (db == null) { session.NewDatabase(IndexRoot.PlaceInDatabase, 0, "IndexRoot"); session.NewDatabase(Lexicon.PlaceInDatabase, 0, "Lexicon"); session.NewDatabase(Document.PlaceInDatabase, 0, "Document"); session.NewDatabase(Repository.PlaceInDatabase, 0, "Repository"); session.NewDatabase(DocumentText.PlaceInDatabase, 0, "DocumentText"); session.NewDatabase(Word.PlaceInDatabase, 0, "Word"); indexRoot = new IndexRoot(btreeNodeSize, session); if (Directory.Exists(s_booksDir)) { string[] directoryTextFiles = Directory.GetFiles(s_booksDir, "*.txt"); foreach (string fileName in directoryTextFiles) { listBoxPagesToAdd.Items.Add(fileName); } } else { wordMinCt.Text = 1.ToString(); listBoxPagesToAdd.Items.Add("http://www.VelocityDB.com/"); // other database products listBoxPagesToAdd.Items.Add("https://foundationdb.com/"); listBoxPagesToAdd.Items.Add("http://www.oracle.com/us/products/database/index.html"); listBoxPagesToAdd.Items.Add("http://www-01.ibm.com/software/data/db2/"); listBoxPagesToAdd.Items.Add("http://www.versant.com/"); listBoxPagesToAdd.Items.Add("http://web.progress.com/en/objectstore/"); listBoxPagesToAdd.Items.Add("https://www.mongodb.org/"); listBoxPagesToAdd.Items.Add("http://cassandra.apache.org/"); listBoxPagesToAdd.Items.Add("http://www.sybase.com/"); listBoxPagesToAdd.Items.Add("http://www.mcobject.com/perst"); listBoxPagesToAdd.Items.Add("http://www.marklogic.com/what-is-marklogic/"); listBoxPagesToAdd.Items.Add("http://hamsterdb.com/"); listBoxPagesToAdd.Items.Add("http://www.firebirdsql.org/"); listBoxPagesToAdd.Items.Add("http://www.h2database.com/"); listBoxPagesToAdd.Items.Add("http://www.oracle.com/technology/products/berkeley-db"); listBoxPagesToAdd.Items.Add("http://www.scimore.com/"); listBoxPagesToAdd.Items.Add("http://www.stsdb.com/"); listBoxPagesToAdd.Items.Add("http://www.sqlite.org/about.html"); listBoxPagesToAdd.Items.Add("http://www.mysql.com/products/enterprise/techspec.html"); listBoxPagesToAdd.Items.Add("http://www.objectivity.com"); listBoxPagesToAdd.Items.Add("http://vistadb.net/"); listBoxPagesToAdd.Items.Add("http://www.google.com/search?q=object+database&sourceid=ie7&rls=com.microsoft:en-us:IE-SearchBox&ie=&oe="); } indexRoot.Persist(session, indexRoot); } else { indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1)); } if (indexRoot.repository.documentSet.Count > 0) { List <Document> docs = indexRoot.repository.documentSet.ToList <Document>().Take(50).ToList <Document>(); inDbListBox.ItemsSource = docs; } updateDataGrids(indexRoot); session.Commit(); //verify(); }
static void Main(string[] args) { long triangles = 0; try { using (SessionNoServer session = new SessionNoServer(systemDir)) { int numberOfWorkerThreads = -1; if (args.Length > 0) { if (!int.TryParse(args[0], out numberOfWorkerThreads)) Console.WriteLine("First parameter is numberOfWorkerThreads which must be an Int32"); } bool useLinq = args.Length > 1; Placement btreePlace = new Placement(40, 1, 1, 10000, 65500, true); Placement edgeInfoPlace = new Placement(40, 10000, 1, 10000, 65500, true); session.BeginUpdate(); BTreeMap<int, int[]> edges; BTreeMapIterator<int, int[]> edgesItr; int[] edge = null; Database edgeDb = session.OpenDatabase(40, false, false); if (edgeDb != null) { session.Commit(); session.BeginRead(); edges = (BTreeMap<int, int[]>)session.Open(40, 1, 1, false); } else { DatabaseLocation location = session.DatabaseLocations.Default(); //location.CompressPages = false; // no compression should make it faster (?) session.NewDatabase(40, 395, "Edges"); edges = new BTreeMap<int, int[]>(null, session, 6000); edges.Persist(btreePlace, session, true); edgesItr = edges.Iterator(); using (StreamReader stream = new StreamReader(edgesInputFile, true)) { int a; int b; string line; string[] fields; while ((line = stream.ReadLine()) != null) { fields = line.Split(' '); if (!int.TryParse(fields[0], out a)) break; b = int.Parse(fields[1]); if (a != b) { if (edgesItr.CurrentKey() == a || edgesItr.GoTo(a)) { edge = edgesItr.CurrentValue(); Array.Resize(ref edge, edge.Length + 1); edge[edge.Length - 1] = b; edgesItr.ReplaceValue(ref edge); // we need to update the value in the BTreeMap } else { edge = new int[1]; edge[0] = b; edges.Add(a, edge); } } } } edgesItr = edges.Iterator(); while (edgesItr.MoveNext()) { edge = edgesItr.CurrentValue(); Array.Sort(edge); edgesItr.ReplaceValue(ref edge); } session.Commit(); session.BeginRead(); } Console.WriteLine("Number of Nodes found: " + edges.Count); if (useLinq) Console.WriteLine("Query using LINQ"); if (numberOfWorkerThreads > 0) Console.WriteLine("Start of triangle discovery using " + numberOfWorkerThreads + " threads, time is " + DateTime.Now); else if (numberOfWorkerThreads < 0) Console.WriteLine("Start of triangle discovery using system automatically selected number of threads, time is " + DateTime.Now); else Console.WriteLine("Start of triangle discovery using main thread, time is " + DateTime.Now); // Start counting triangles ! if (numberOfWorkerThreads != 0) { if (useLinq) { // Please help, I have not figured out how to properly do the triangle query using LINQ int[] edge2values = null; if (numberOfWorkerThreads > 0) triangles = (from KeyValuePair<int, int[]> edgeFrom in edges from int edgeTo1 in edgeFrom.Value where edgeFrom.Key < edgeTo1 from int edgeTo2 in edgeFrom.Value where edgeFrom.Key < edgeTo2 && edgeTo2 > edgeTo1 && edges.TryGetValue(edgeTo1, out edge2values) && Array.BinarySearch(edge2values, edgeTo2) >= 0 select edge).AsParallel().WithDegreeOfParallelism(numberOfWorkerThreads).Count(); else triangles = (from KeyValuePair<int, int[]> edgeFrom in edges from int edgeTo1 in edgeFrom.Value where edgeFrom.Key < edgeTo1 from int edgeTo2 in edgeFrom.Value where edgeFrom.Key < edgeTo2 && edgeTo2 > edgeTo1 && edges.TryGetValue(edgeTo1, out edge2values) && Array.BinarySearch(edge2values, edgeTo2) >= 0 select edge).AsParallel().Count(); } else { edgesItr = edges.Iterator(); ParallelOptions pOptions = new ParallelOptions(); pOptions.MaxDegreeOfParallelism = numberOfWorkerThreads; // First type parameter is the type of the source elements // Second type parameter is the type of the local data (subtotal) Parallel.ForEach<KeyValuePair<int, int[]>, long>(edges, // source collection pOptions, () => 0, // method to initialize the local variable (pair, loop, subtotal) => // method invoked by the loop on each iteration { int nodeId = pair.Key; int[] nodeTo = pair.Value; int stop = nodeTo.Length - 1; int i = stop; int edgeToStart, edgeTo; int pos; while (i >= 0) { int[] edgeInfo2; edgeToStart = nodeTo[i--]; if (nodeId < edgeToStart) { if (edges.TryGetValue(edgeToStart, out edgeInfo2)) { for (int j = stop; j >= i; j--) { edgeTo = nodeTo[j]; if (edgeToStart < edgeTo) { pos = Array.BinarySearch<int>(edgeInfo2, edgeTo); if (pos >= 0) { // we know this one is connected to edgeInfo.From because it is part of edgeInfo.To subtotal++; } } else break; } } } else break; } return subtotal; }, // Method to be executed when all loops have completed. // finalResult is the final value of subtotal. supplied by the ForEach method. (finalResult) => Interlocked.Add(ref triangles, finalResult)); } } else if (useLinq) triangles = queryUsingLINQ(edges); else triangles = discoverTrianglesSingleCore(edges); session.Commit(); } Console.WriteLine("Number of Triangles found: " + triangles); } catch (Exception e) { System.Console.WriteLine(e); } }