private void button1_Click(object sender, RoutedEventArgs e) { session.BeginUpdate(); IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1)); Placement docPlacement = new Placement(Document.PlaceInDatabase); foreach (string str in listBoxPagesToAdd.Items) { Document doc = null; try { if (str.Contains(".html") || str.Contains(".htm") || str.Contains("http") || str.Contains("aspx")) { doc = parseHtml(str, indexRoot); } else { doc = parseTextFile(str, indexRoot, docPlacement); } } catch (WebException ex) { Console.WriteLine(ex.ToString()); } } createGlobalInvertedIndex(indexRoot); listBoxPagesToAdd.Items.Clear(); List <Document> docs = indexRoot.repository.documentSet.ToList <Document>().Take(50).ToList <Document>(); inDbListBox.ItemsSource = docs; session.Commit(); session.BeginRead(); updateDataGrids(indexRoot); session.Commit(); }
static void createInvertedIndex() { using (SessionNoServer session = new SessionNoServer(s_systemDir)) { session.BeginUpdate(); session.EnableAutoPageFlush = false; // so that threads don't stomb on each other Console.WriteLine(DateTime.Now.ToString() + ", start creating inverted index"); ParallelOptions pOptions = new ParallelOptions(); pOptions.MaxDegreeOfParallelism = 2; // set to what is appropriate for your computer (cores & memory size) //pOptions.MaxDegreeOfParallelism = 1; // appears to work best with only 16GB of memory IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1)); BTreeSet <Document> documentSet = indexRoot.repository.documentSet; List <Database> dbs = session.OpenAllDatabases(true); Parallel.ForEach <Database>(dbs, pOptions, (Database db, ParallelLoopState loop) => // method invoked by the loop on each iteration { if (db.DatabaseNumber >= Document.PlaceInDatabase) { createDocumentInvertedIndex(session, db, documentSet); } }); session.Commit(); Console.WriteLine(DateTime.Now.ToString() + ", done creating inverted index"); } }
public void LookupCompareFields(int bTreeDatabaseNumber) { using (SessionNoServer session = new SessionNoServer(systemDir)) { Person personPrior = null, person; session.BeginRead(); BTreeSet <Person> bTree = (BTreeSet <Person>)session.Open(Oid.Encode((uint)bTreeDatabaseNumber, 1, 1)); BTreeSetIterator <Person> itr = bTree.Iterator(); int ct = 0; while ((person = itr.Next()) != null) { if (personPrior != null) { Assert.LessOrEqual(personPrior.FirstName, person.FirstName); } ct++; personPrior = person; } int ct2 = 0; personPrior = null; foreach (Person pers in (IEnumerable <Person>)bTree) { if (personPrior != null) { Assert.LessOrEqual(personPrior.FirstName, pers.FirstName); } ct2++; personPrior = pers; } session.Commit(); Assert.AreEqual(ct, ct2); } }
public void UnpersistCompareFields(int bTreeDatabaseNumber) { using (SessionNoServer session = new SessionNoServer(systemDir)) { session.BeginUpdate(); BTreeSet <Person> bTree = (BTreeSet <Person>)session.Open(Oid.Encode((uint)bTreeDatabaseNumber, 1, 1)); BTreeSetIterator <Person> itr = bTree.Iterator(); itr.GoToLast(); itr.Remove(); session.Abort(); session.BeginUpdate(); bTree = (BTreeSet <Person>)session.Open(Oid.Encode((uint)bTreeDatabaseNumber, 1, 1)); bTree.Unpersist(session); session.Commit(); session.BeginRead(); Database db = session.OpenDatabase((uint)bTreeDatabaseNumber, false); foreach (Page page in db) { foreach (OptimizedPersistable obj in page) { if (obj.PageNumber > 0) { Assert.Fail("No objects should remain in this database"); } } } session.Commit(); } }
private void db2UpdateSession1_Click(object sender, RoutedEventArgs e) { if (thread[0] != null && thread[0].IsAlive) { thread[0].Join(); } thread[0] = new Thread(() => { try { Number pObj = (Number)session[0].Open(Oid.Encode(11, 1, 1), true, null, false, 0, 1); pObj.Update(); Application.Current.Dispatcher.Invoke(new Action(() => { session1objectDb1.Content = pObj.ToString(); session1messages.Content = ""; })); } catch (Exception ex) { Application.Current.Dispatcher.Invoke(new Action(() => { session1messages.Content = ex.Message; })); } }); thread[0].Start(); }
private void MenuItem_Click(object sender, RoutedEventArgs e) { MenuItem menuItem = (MenuItem)e.Source; Document myItem = (Document)menuItem.DataContext; session.BeginUpdate(); try { IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1)); int index; if (indexRoot.repository.documentSet.TryGetKey(myItem, ref myItem)) { index = myItem.Remove(indexRoot, session); } else { index = -1; // weird case - should not happen } inDbListBox.ItemsSource = indexRoot.repository.documentSet.ToList <Document>(); updateDataGrids(indexRoot, index); session.Commit(); } catch { session.Abort(); } }
static void createTopLevelInvertedIndex() { Console.WriteLine(DateTime.Now.ToString() + ", start creating top level inverted index"); using (SessionNoServer session = new SessionNoServer(s_systemDir)) { Placement wordPlacement = new Placement(Lexicon.PlaceInDatabase, 2, 1, 1000, 50000, true, false, UInt32.MaxValue, false); session.BeginUpdate(); IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1)); BTreeSetOidShort <Word> wordSet = indexRoot.lexicon.WordSet; BTreeSet <Document> documentSet = indexRoot.repository.documentSet; Word existingWord = null; foreach (Document doc in documentSet) { foreach (Word word in doc.WordSet) { WordHit wordHit = doc.WordHit[word]; if (wordSet.TryGetKey(word, ref existingWord)) { existingWord.GlobalCount = existingWord.GlobalCount + (uint)wordHit.Count; } else { existingWord = new WordGlobal(word.aWord, session, (uint)wordHit.Count); existingWord.Persist(wordPlacement, session); indexRoot.lexicon.WordSet.Add(existingWord); } existingWord.DocumentHit.AddFast(doc); } doc.Indexed = true; } session.Commit(); Console.WriteLine(DateTime.Now.ToString() + ", done creating top level inverted index"); } }
public static bool IsOidStr(string inStr) { bool retval = false; MemoryStream ms = new MemoryStream(); Oid xoid = new Oid(); xoid.Encode(ms, inStr); ms.Close(); retval = true; return(retval); }
private void db1UpdateSession4_Click(object sender, RoutedEventArgs e) { try { Number pObj = (Number)session[3].Open(Oid.Encode(10, 1, 1), true, null, false, 0, 1); session4objectDb1.Content = pObj.ToString(); session4messages.Content = ""; } catch (Exception ex) { session4messages.Content = ex.Message; } }
private void db3ReadSession3_Click(object sender, RoutedEventArgs e) { try { Number pObj = (Number)session[2].Open(Oid.Encode(12, 1, 1), false, null, false, 0, 1); session3objectDb3.Content = pObj.ToString(); session3messages.Content = ""; } catch (Exception ex) { session3messages.Content = ex.Message; } }
private void add1db2Session3_Click(object sender, RoutedEventArgs e) { try { Number pObj = (Number)session[2].Open(Oid.Encode(11, 1, 1), true, null, false, 0, 1); pObj.MyInt = pObj.MyInt + 1; session3objectDb2.Content = pObj.ToString(); session3messages.Content = ""; } catch (Exception ex) { session3messages.Content = ex.Message; } }
static void outputSomeInfo(SessionNoServer session) { IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1)); BTreeSetOidShort <Word> wordSet = indexRoot.lexicon.WordSet; using (StreamWriter writer = new StreamWriter("Wikipedia.txt")) { writer.WriteLine("Number of words in Lexicon is: " + wordSet.Count); foreach (Word word in wordSet) { writer.WriteLine(word.aWord + " " + word.DocumentHit.Count); } writer.Close(); } }
private void updateWordTables_Click(object sender, RoutedEventArgs e) { if (session != null) { try { session.BeginRead(); IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1)); updateDataGrids(indexRoot); } finally { session.Commit(); } } }
static void outputSomeInfo(SessionNoServer session) { IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1)); var wordHits = indexRoot.Lexicon.TokenMap; using (StreamWriter writer = new StreamWriter("Wikipedia.txt")) { writer.WriteLine("Number of words in Lexicon is: " + indexRoot.Lexicon.IdToValue.Count); foreach (var p in wordHits) { var word = indexRoot.Lexicon.IdToValue[p.Key]; writer.WriteLine(word + " " + p.Value.Count); } writer.Close(); } }
static void CreateInvertedIndex() { using (SessionNoServer session = new SessionNoServer(s_systemDir, 5000, false, false, CacheEnum.No)) // turn of page and object caching)) { session.BeginUpdate(); session.RegisterClass(typeof(Repository)); session.RegisterClass(typeof(IndexRoot)); session.RegisterClass(typeof(Lexicon <string>)); Console.WriteLine(DateTime.Now.ToString() + ", start creating inverted index"); IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1)); BTreeSet <Document> documentSet = indexRoot.Repository.DocumentSet; foreach (var doc in documentSet) { createDocumentInvertedIndex(indexRoot, doc); } session.Commit(); Console.WriteLine(DateTime.Now.ToString() + ", done creating inverted index"); } }
public void verify(bool startTrans = true) { if (startTrans) { session.BeginRead(); } IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1)); int i = 0; int j = 0; int k = 0; foreach (Word word in indexRoot.lexicon.WordSet) { i++; foreach (Document doc in word.DocumentHit) { j++; if (doc == null) { throw new UnexpectedException("bad documentHit BTreeSet"); } foreach (KeyValuePair <Word, WordHit> pair in doc.WordHit) { k++; if (pair.Value == null || pair.Key == null) { throw new UnexpectedException("bad document WordHit"); } if (indexRoot.lexicon.WordSet.Contains(pair.Key) == false) { throw new UnexpectedException("missing lexicon word"); } } } } if (startTrans) { session.Commit(); } }
public void verify(bool startTrans = true) { if (startTrans) { session.BeginRead(); } IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1)); int i = 0; int j = 0; int k = 0; foreach (var pair in indexRoot.Lexicon.TokenMap) { i++; foreach (Document doc in pair.Value) { j++; if (doc == null) { throw new UnexpectedException("bad documentHit BTreeSet"); } foreach (KeyValuePair <UInt32, UInt32> pair2 in doc.WordHit) { k++; if (pair2.Value == 0) { throw new UnexpectedException("bad document WordHit"); } if (indexRoot.Lexicon.IdToValue.Contains(pair2.Key) == false) { throw new UnexpectedException("missing lexicon word"); } } } } if (startTrans) { session.Commit(); } }
static public bool EditNode(IWin32Window parent, Asn1Node aNode, bool enableTagEdit, bool pureHexMode) { byte[] val; byte[] data; FormNodeContentEditor ed = new FormNodeContentEditor(); ed.aNode = aNode; MemoryStream ms = new MemoryStream(); ed.checker = FormNodeContentEditor.DataChecker.None; ed.enableTagEdit = enableTagEdit; ed.pureHexMode = pureHexMode; if ( ((aNode.Tag & Asn1Tag.TAG_MASK) == Asn1Tag.BIT_STRING) && (aNode.ChildNodeCount < 1)) { ed.panelUnusedBits.Visible = true; ed.textBoxUnusedBits.Text = aNode.UnusedBits.ToString(); } else { ed.panelUnusedBits.Visible = false; } if (pureHexMode) { ed.checker = DataChecker.Hex; ed.ShowDialog(parent); if (!ed.isOK) { return(false); } data = Asn1Util.HexStrToBytes(ed.GetValueStr()); aNode.Data = data; } else { byte[] oidVal; switch (aNode.Tag) { case Asn1Tag.OBJECT_IDENTIFIER: ed.checker = DataChecker.Oid; ed.ShowDialog(parent); if (!ed.isOK) { return(false); } Oid xoid = new Oid(); xoid.Encode(ms, ed.GetValueStr()); ms.Position = 0; oidVal = new byte[ms.Length]; ms.Read(oidVal, 0, (int)ms.Length); ms.Close(); aNode.Data = oidVal; break; case Asn1Tag.RELATIVE_OID: ed.checker = DataChecker.Roid; ed.ShowDialog(parent); if (!ed.isOK) { return(false); } RelativeOid roid = new RelativeOid(); roid.Encode(ms, ed.GetValueStr()); ms.Position = 0; oidVal = new byte[ms.Length]; ms.Read(oidVal, 0, (int)ms.Length); ms.Close(); aNode.Data = oidVal; break; case Asn1Tag.PRINTABLE_STRING: case Asn1Tag.IA5_STRING: case Asn1Tag.UNIVERSAL_STRING: case Asn1Tag.VISIBLE_STRING: case Asn1Tag.NUMERIC_STRING: case Asn1Tag.UTC_TIME: case Asn1Tag.GENERAL_STRING: case Asn1Tag.GENERALIZED_TIME: ed.ShowDialog(parent); if (!ed.isOK) { return(false); } val = Asn1Util.StringToBytes(ed.GetValueStr()); aNode.Data = val; break; case Asn1Tag.UTF8_STRING: ed.ShowDialog(parent); if (!ed.isOK) { return(false); } UTF8Encoding u8 = new UTF8Encoding(false); val = u8.GetBytes(ed.GetValueStr()); aNode.Data = val; break; case Asn1Tag.BMPSTRING: ed.ShowDialog(parent); if (!ed.isOK) { return(false); } //byte[] tmpval = Asn1Util.StringToBytes(ed.GetValueStr()); byte[] tmpval = Encoding.BigEndianUnicode.GetBytes(ed.GetValueStr()); val = new byte[tmpval.Length * 2]; for (int i = 0; i < tmpval.Length; i++) { val[i * 2] = 0; val[i * 2 + 1] = tmpval[i]; } aNode.Data = val; break; case Asn1Tag.INTEGER: case Asn1Tag.BIT_STRING: ed.checker = DataChecker.Hex; ed.ShowDialog(parent); if (!ed.isOK) { return(false); } aNode.UnusedBits = (byte)(Convert.ToUInt16(ed.textBoxUnusedBits.Text) % 8); data = Asn1Util.HexStrToBytes(ed.GetValueStr()); aNode.Data = data; break; default: if ((aNode.Tag & Asn1Tag.TAG_MASK) == 6) // Visible string for certificate { ed.ShowDialog(parent); if (!ed.isOK) { return(false); } val = Asn1Util.StringToBytes(ed.GetValueStr()); aNode.Data = val; } else { ed.checker = DataChecker.Hex; ed.ShowDialog(parent); if (!ed.isOK) { return(false); } data = Asn1Util.HexStrToBytes(ed.GetValueStr()); aNode.Data = data; } break; } ; } return(true); }
public void SimpleApiServer() { long ssn = 555555; UInt16 age = 1; VelocityDbSchema.Person mats; Placement place; using (ServerClientSession session = new ServerClientSession(systemDir)) { // skip delete database since it invalidates indices session.BeginUpdate(); Database db = session.OpenDatabase(10, true, false); if (db != null) { session.DeleteDatabase(db); } session.Commit(); session.BeginUpdate(); place = new Placement(10, 2, 1, 1, 10); DateTime birthday = new DateTime(1960, 6, 13); mats = new Man("Mats", "Persson", age++, ssn++, birthday); mats.Persist(place, session); session.Commit(); session.ClearServerCache(); } UInt64 mOid1 = Oid.Encode(10, 2, 1); using (ServerClientSession session = new ServerClientSession(systemDir)) { session.BeginUpdate(); UInt32 dbNum = Oid.DatabaseNumber(mOid1); mats = (VelocityDbSchema.Person)session.Open(mOid1); Woman kinga = null; mats = new Man("Mats", "Persson", age++, ssn++, new DateTime(1960, 6, 13)); Cat cat = new Cat("Boze", 8); mats.m_pets.Add(cat); Bird bird = new Bird("Pippi", 1); cat.friends.Add(bird); mats.Persist(place, session); kinga = new Woman("Kinga", "Persson", age, ssn, mats, mats); kinga.Persist(place, session); VelocityDbSchema.Person robin = new VelocityDbSchema.Person("Robin", "Persson", 13, 1, mats, null); robin.Persist(place, session); mOid1 = mats.Id; mats = null; mats = (VelocityDbSchema.Person)session.Open(mOid1); session.Commit(); session.ClearServerCache(); } using (ServerClientSession session = new ServerClientSession(systemDir)) { session.BeginUpdate(); mats = (VelocityDbSchema.Person)session.Open(mOid1); session.Commit(); session.ClearServerCache(); } using (ServerClientSession session = new ServerClientSession(systemDir)) { session.BeginRead(); ulong mOid2 = mats.Id; mats = (VelocityDbSchema.Person)session.Open(mOid2); session.Commit(); session.ClearServerCache(); } }
public MainWindow() { const ushort btreeNodeSize = 5000; GCSettings.LatencyMode = GCLatencyMode.Batch;// try to keep the WeakIOptimizedPersistableReference objects around longer dataGridList = new List <DataGrid>(); dataTableList = new List <DataTable>(); InitializeComponent(); session = new SessionNoServer(s_systemDir); Placement placerIndexRoot = new Placement(IndexRoot.PlaceInDatabase); session.BeginUpdate(); Console.WriteLine("Running with databases in directory: " + session.SystemDirectory); File.Copy(s_licenseDbFile, Path.Combine(session.SystemDirectory, "4.odb"), true); IndexRoot indexRoot; Database db = session.OpenDatabase(IndexRoot.PlaceInDatabase, false, false); if (db == null) { session.NewDatabase(IndexRoot.PlaceInDatabase, 0, "IndexRoot"); session.NewDatabase(Lexicon.PlaceInDatabase, 0, "Lexicon"); session.NewDatabase(Document.PlaceInDatabase, 0, "Document"); session.NewDatabase(Repository.PlaceInDatabase, 0, "Repository"); session.NewDatabase(DocumentText.PlaceInDatabase, 0, "DocumentText"); session.NewDatabase(Word.PlaceInDatabase, 0, "Word"); indexRoot = new IndexRoot(btreeNodeSize, session); if (Directory.Exists(s_booksDir)) { string[] directoryTextFiles = Directory.GetFiles(s_booksDir, "*.txt"); foreach (string fileName in directoryTextFiles) { listBoxPagesToAdd.Items.Add(fileName); } } else { wordMinCt.Text = 1.ToString(); listBoxPagesToAdd.Items.Add("http://www.VelocityDB.com/"); // other database products listBoxPagesToAdd.Items.Add("https://foundationdb.com/"); listBoxPagesToAdd.Items.Add("http://www.oracle.com/us/products/database/index.html"); listBoxPagesToAdd.Items.Add("http://www-01.ibm.com/software/data/db2/"); listBoxPagesToAdd.Items.Add("http://www.versant.com/"); listBoxPagesToAdd.Items.Add("http://web.progress.com/en/objectstore/"); listBoxPagesToAdd.Items.Add("https://www.mongodb.org/"); listBoxPagesToAdd.Items.Add("http://cassandra.apache.org/"); listBoxPagesToAdd.Items.Add("http://www.sybase.com/"); listBoxPagesToAdd.Items.Add("http://www.mcobject.com/perst"); listBoxPagesToAdd.Items.Add("http://www.marklogic.com/what-is-marklogic/"); listBoxPagesToAdd.Items.Add("http://hamsterdb.com/"); listBoxPagesToAdd.Items.Add("http://www.firebirdsql.org/"); listBoxPagesToAdd.Items.Add("http://www.h2database.com/"); listBoxPagesToAdd.Items.Add("http://www.oracle.com/technology/products/berkeley-db"); listBoxPagesToAdd.Items.Add("http://www.scimore.com/"); listBoxPagesToAdd.Items.Add("http://www.stsdb.com/"); listBoxPagesToAdd.Items.Add("http://www.sqlite.org/about.html"); listBoxPagesToAdd.Items.Add("http://www.mysql.com/products/enterprise/techspec.html"); listBoxPagesToAdd.Items.Add("http://www.objectivity.com"); listBoxPagesToAdd.Items.Add("http://vistadb.net/"); listBoxPagesToAdd.Items.Add("http://www.google.com/search?q=object+database&sourceid=ie7&rls=com.microsoft:en-us:IE-SearchBox&ie=&oe="); } indexRoot.Persist(session, indexRoot); } else { indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1)); } if (indexRoot.repository.documentSet.Count > 0) { List <Document> docs = indexRoot.repository.documentSet.ToList <Document>().Take(50).ToList <Document>(); inDbListBox.ItemsSource = docs; } updateDataGrids(indexRoot); session.Commit(); //verify(); }