private void button1_Click(object sender, RoutedEventArgs e)
{
session.BeginUpdate();
IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1));
Placement docPlacement = new Placement(Document.PlaceInDatabase);
foreach (string str in listBoxPagesToAdd.Items)
{
Document doc = null;
try
{
if (str.Contains(".html") || str.Contains(".htm") || str.Contains("http") || str.Contains("aspx"))
{
doc = parseHtml(str, indexRoot);
}
else
{
doc = parseTextFile(str, indexRoot, docPlacement);
}
}
catch (WebException ex)
{
Console.WriteLine(ex.ToString());
}
}
createGlobalInvertedIndex(indexRoot);
listBoxPagesToAdd.Items.Clear();
List <Document> docs = indexRoot.repository.documentSet.ToList <Document>().Take(50).ToList <Document>();
inDbListBox.ItemsSource = docs;
session.Commit();
session.BeginRead();
updateDataGrids(indexRoot);
session.Commit();
}
static void createInvertedIndex()
{
using (SessionNoServer session = new SessionNoServer(s_systemDir))
{
session.BeginUpdate();
session.EnableAutoPageFlush = false; // so that threads don't stomb on each other
Console.WriteLine(DateTime.Now.ToString() + ", start creating inverted index");
ParallelOptions pOptions = new ParallelOptions();
pOptions.MaxDegreeOfParallelism = 2; // set to what is appropriate for your computer (cores & memory size)
//pOptions.MaxDegreeOfParallelism = 1; // appears to work best with only 16GB of memory
IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1));
BTreeSet <Document> documentSet = indexRoot.repository.documentSet;
List <Database> dbs = session.OpenAllDatabases(true);
Parallel.ForEach <Database>(dbs, pOptions,
(Database db, ParallelLoopState loop) => // method invoked by the loop on each iteration
{
if (db.DatabaseNumber >= Document.PlaceInDatabase)
{
createDocumentInvertedIndex(session, db, documentSet);
}
});
session.Commit();
Console.WriteLine(DateTime.Now.ToString() + ", done creating inverted index");
}
}
public void LookupCompareFields(int bTreeDatabaseNumber)
{
using (SessionNoServer session = new SessionNoServer(systemDir))
{
Person personPrior = null, person;
session.BeginRead();
BTreeSet <Person> bTree = (BTreeSet <Person>)session.Open(Oid.Encode((uint)bTreeDatabaseNumber, 1, 1));
BTreeSetIterator <Person> itr = bTree.Iterator();
int ct = 0;
while ((person = itr.Next()) != null)
{
if (personPrior != null)
{
Assert.LessOrEqual(personPrior.FirstName, person.FirstName);
}
ct++;
personPrior = person;
}
int ct2 = 0;
personPrior = null;
foreach (Person pers in (IEnumerable <Person>)bTree)
{
if (personPrior != null)
{
Assert.LessOrEqual(personPrior.FirstName, pers.FirstName);
}
ct2++;
personPrior = pers;
}
session.Commit();
Assert.AreEqual(ct, ct2);
}
}
public void UnpersistCompareFields(int bTreeDatabaseNumber)
{
using (SessionNoServer session = new SessionNoServer(systemDir))
{
session.BeginUpdate();
BTreeSet <Person> bTree = (BTreeSet <Person>)session.Open(Oid.Encode((uint)bTreeDatabaseNumber, 1, 1));
BTreeSetIterator <Person> itr = bTree.Iterator();
itr.GoToLast();
itr.Remove();
session.Abort();
session.BeginUpdate();
bTree = (BTreeSet <Person>)session.Open(Oid.Encode((uint)bTreeDatabaseNumber, 1, 1));
bTree.Unpersist(session);
session.Commit();
session.BeginRead();
Database db = session.OpenDatabase((uint)bTreeDatabaseNumber, false);
foreach (Page page in db)
{
foreach (OptimizedPersistable obj in page)
{
if (obj.PageNumber > 0)
{
Assert.Fail("No objects should remain in this database");
}
}
}
session.Commit();
}
}
private void db2UpdateSession1_Click(object sender, RoutedEventArgs e)
{
if (thread[0] != null && thread[0].IsAlive)
{
thread[0].Join();
}
thread[0] = new Thread(() =>
{
try
{
Number pObj = (Number)session[0].Open(Oid.Encode(11, 1, 1), true, null, false, 0, 1);
pObj.Update();
Application.Current.Dispatcher.Invoke(new Action(() =>
{
session1objectDb1.Content = pObj.ToString();
session1messages.Content = "";
}));
}
catch (Exception ex)
{
Application.Current.Dispatcher.Invoke(new Action(() => { session1messages.Content = ex.Message; }));
}
});
thread[0].Start();
}
private void MenuItem_Click(object sender, RoutedEventArgs e)
{
MenuItem menuItem = (MenuItem)e.Source;
Document myItem = (Document)menuItem.DataContext;
session.BeginUpdate();
try
{
IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1));
int index;
if (indexRoot.repository.documentSet.TryGetKey(myItem, ref myItem))
{
index = myItem.Remove(indexRoot, session);
}
else
{
index = -1; // weird case - should not happen
}
inDbListBox.ItemsSource = indexRoot.repository.documentSet.ToList <Document>();
updateDataGrids(indexRoot, index);
session.Commit();
}
catch
{
session.Abort();
}
}
static void createTopLevelInvertedIndex()
{
Console.WriteLine(DateTime.Now.ToString() + ", start creating top level inverted index");
using (SessionNoServer session = new SessionNoServer(s_systemDir))
{
Placement wordPlacement = new Placement(Lexicon.PlaceInDatabase, 2, 1, 1000, 50000, true, false, UInt32.MaxValue, false);
session.BeginUpdate();
IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1));
BTreeSetOidShort <Word> wordSet = indexRoot.lexicon.WordSet;
BTreeSet <Document> documentSet = indexRoot.repository.documentSet;
Word existingWord = null;
foreach (Document doc in documentSet)
{
foreach (Word word in doc.WordSet)
{
WordHit wordHit = doc.WordHit[word];
if (wordSet.TryGetKey(word, ref existingWord))
{
existingWord.GlobalCount = existingWord.GlobalCount + (uint)wordHit.Count;
}
else
{
existingWord = new WordGlobal(word.aWord, session, (uint)wordHit.Count);
existingWord.Persist(wordPlacement, session);
indexRoot.lexicon.WordSet.Add(existingWord);
}
existingWord.DocumentHit.AddFast(doc);
}
doc.Indexed = true;
}
session.Commit();
Console.WriteLine(DateTime.Now.ToString() + ", done creating top level inverted index");
}
}
public static bool IsOidStr(string inStr)
{
bool retval = false;
MemoryStream ms = new MemoryStream();
Oid xoid = new Oid();
xoid.Encode(ms, inStr);
ms.Close();
retval = true;
return(retval);
}
private void db1UpdateSession4_Click(object sender, RoutedEventArgs e)
{
try
{
Number pObj = (Number)session[3].Open(Oid.Encode(10, 1, 1), true, null, false, 0, 1);
session4objectDb1.Content = pObj.ToString();
session4messages.Content = "";
}
catch (Exception ex)
{
session4messages.Content = ex.Message;
}
}
private void db3ReadSession3_Click(object sender, RoutedEventArgs e)
{
try
{
Number pObj = (Number)session[2].Open(Oid.Encode(12, 1, 1), false, null, false, 0, 1);
session3objectDb3.Content = pObj.ToString();
session3messages.Content = "";
}
catch (Exception ex)
{
session3messages.Content = ex.Message;
}
}
private void add1db2Session3_Click(object sender, RoutedEventArgs e)
{
try
{
Number pObj = (Number)session[2].Open(Oid.Encode(11, 1, 1), true, null, false, 0, 1);
pObj.MyInt = pObj.MyInt + 1;
session3objectDb2.Content = pObj.ToString();
session3messages.Content = "";
}
catch (Exception ex)
{
session3messages.Content = ex.Message;
}
}
static void outputSomeInfo(SessionNoServer session)
{
IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1));
BTreeSetOidShort <Word> wordSet = indexRoot.lexicon.WordSet;
using (StreamWriter writer = new StreamWriter("Wikipedia.txt"))
{
writer.WriteLine("Number of words in Lexicon is: " + wordSet.Count);
foreach (Word word in wordSet)
{
writer.WriteLine(word.aWord + " " + word.DocumentHit.Count);
}
writer.Close();
}
}
private void updateWordTables_Click(object sender, RoutedEventArgs e)
{
if (session != null)
{
try
{
session.BeginRead();
IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1));
updateDataGrids(indexRoot);
}
finally
{
session.Commit();
}
}
}
static void outputSomeInfo(SessionNoServer session)
{
IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1));
var wordHits = indexRoot.Lexicon.TokenMap;
using (StreamWriter writer = new StreamWriter("Wikipedia.txt"))
{
writer.WriteLine("Number of words in Lexicon is: " + indexRoot.Lexicon.IdToValue.Count);
foreach (var p in wordHits)
{
var word = indexRoot.Lexicon.IdToValue[p.Key];
writer.WriteLine(word + " " + p.Value.Count);
}
writer.Close();
}
}
static void CreateInvertedIndex()
{
using (SessionNoServer session = new SessionNoServer(s_systemDir, 5000, false, false, CacheEnum.No)) // turn of page and object caching))
{
session.BeginUpdate();
session.RegisterClass(typeof(Repository));
session.RegisterClass(typeof(IndexRoot));
session.RegisterClass(typeof(Lexicon <string>));
Console.WriteLine(DateTime.Now.ToString() + ", start creating inverted index");
IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1));
BTreeSet <Document> documentSet = indexRoot.Repository.DocumentSet;
foreach (var doc in documentSet)
{
createDocumentInvertedIndex(indexRoot, doc);
}
session.Commit();
Console.WriteLine(DateTime.Now.ToString() + ", done creating inverted index");
}
}
public void verify(bool startTrans = true)
{
if (startTrans)
{
session.BeginRead();
}
IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1));
int i = 0;
int j = 0;
int k = 0;
foreach (Word word in indexRoot.lexicon.WordSet)
{
i++;
foreach (Document doc in word.DocumentHit)
{
j++;
if (doc == null)
{
throw new UnexpectedException("bad documentHit BTreeSet");
}
foreach (KeyValuePair <Word, WordHit> pair in doc.WordHit)
{
k++;
if (pair.Value == null || pair.Key == null)
{
throw new UnexpectedException("bad document WordHit");
}
if (indexRoot.lexicon.WordSet.Contains(pair.Key) == false)
{
throw new UnexpectedException("missing lexicon word");
}
}
}
}
if (startTrans)
{
session.Commit();
}
}
public void verify(bool startTrans = true)
{
if (startTrans)
{
session.BeginRead();
}
IndexRoot indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1));
int i = 0;
int j = 0;
int k = 0;
foreach (var pair in indexRoot.Lexicon.TokenMap)
{
i++;
foreach (Document doc in pair.Value)
{
j++;
if (doc == null)
{
throw new UnexpectedException("bad documentHit BTreeSet");
}
foreach (KeyValuePair <UInt32, UInt32> pair2 in doc.WordHit)
{
k++;
if (pair2.Value == 0)
{
throw new UnexpectedException("bad document WordHit");
}
if (indexRoot.Lexicon.IdToValue.Contains(pair2.Key) == false)
{
throw new UnexpectedException("missing lexicon word");
}
}
}
}
if (startTrans)
{
session.Commit();
}
}
static public bool EditNode(IWin32Window parent, Asn1Node aNode, bool enableTagEdit, bool pureHexMode)
{
byte[] val;
byte[] data;
FormNodeContentEditor ed = new FormNodeContentEditor();
ed.aNode = aNode;
MemoryStream ms = new MemoryStream();
ed.checker = FormNodeContentEditor.DataChecker.None;
ed.enableTagEdit = enableTagEdit;
ed.pureHexMode = pureHexMode;
if (
((aNode.Tag & Asn1Tag.TAG_MASK) == Asn1Tag.BIT_STRING) &&
(aNode.ChildNodeCount < 1))
{
ed.panelUnusedBits.Visible = true;
ed.textBoxUnusedBits.Text = aNode.UnusedBits.ToString();
}
else
{
ed.panelUnusedBits.Visible = false;
}
if (pureHexMode)
{
ed.checker = DataChecker.Hex;
ed.ShowDialog(parent);
if (!ed.isOK)
{
return(false);
}
data = Asn1Util.HexStrToBytes(ed.GetValueStr());
aNode.Data = data;
}
else
{
byte[] oidVal;
switch (aNode.Tag)
{
case Asn1Tag.OBJECT_IDENTIFIER:
ed.checker = DataChecker.Oid;
ed.ShowDialog(parent);
if (!ed.isOK)
{
return(false);
}
Oid xoid = new Oid();
xoid.Encode(ms, ed.GetValueStr());
ms.Position = 0;
oidVal = new byte[ms.Length];
ms.Read(oidVal, 0, (int)ms.Length);
ms.Close();
aNode.Data = oidVal;
break;
case Asn1Tag.RELATIVE_OID:
ed.checker = DataChecker.Roid;
ed.ShowDialog(parent);
if (!ed.isOK)
{
return(false);
}
RelativeOid roid = new RelativeOid();
roid.Encode(ms, ed.GetValueStr());
ms.Position = 0;
oidVal = new byte[ms.Length];
ms.Read(oidVal, 0, (int)ms.Length);
ms.Close();
aNode.Data = oidVal;
break;
case Asn1Tag.PRINTABLE_STRING:
case Asn1Tag.IA5_STRING:
case Asn1Tag.UNIVERSAL_STRING:
case Asn1Tag.VISIBLE_STRING:
case Asn1Tag.NUMERIC_STRING:
case Asn1Tag.UTC_TIME:
case Asn1Tag.GENERAL_STRING:
case Asn1Tag.GENERALIZED_TIME:
ed.ShowDialog(parent);
if (!ed.isOK)
{
return(false);
}
val = Asn1Util.StringToBytes(ed.GetValueStr());
aNode.Data = val;
break;
case Asn1Tag.UTF8_STRING:
ed.ShowDialog(parent);
if (!ed.isOK)
{
return(false);
}
UTF8Encoding u8 = new UTF8Encoding(false);
val = u8.GetBytes(ed.GetValueStr());
aNode.Data = val;
break;
case Asn1Tag.BMPSTRING:
ed.ShowDialog(parent);
if (!ed.isOK)
{
return(false);
}
//byte[] tmpval = Asn1Util.StringToBytes(ed.GetValueStr());
byte[] tmpval = Encoding.BigEndianUnicode.GetBytes(ed.GetValueStr());
val = new byte[tmpval.Length * 2];
for (int i = 0; i < tmpval.Length; i++)
{
val[i * 2] = 0;
val[i * 2 + 1] = tmpval[i];
}
aNode.Data = val;
break;
case Asn1Tag.INTEGER:
case Asn1Tag.BIT_STRING:
ed.checker = DataChecker.Hex;
ed.ShowDialog(parent);
if (!ed.isOK)
{
return(false);
}
aNode.UnusedBits = (byte)(Convert.ToUInt16(ed.textBoxUnusedBits.Text) % 8);
data = Asn1Util.HexStrToBytes(ed.GetValueStr());
aNode.Data = data;
break;
default:
if ((aNode.Tag & Asn1Tag.TAG_MASK) == 6) // Visible string for certificate
{
ed.ShowDialog(parent);
if (!ed.isOK)
{
return(false);
}
val = Asn1Util.StringToBytes(ed.GetValueStr());
aNode.Data = val;
}
else
{
ed.checker = DataChecker.Hex;
ed.ShowDialog(parent);
if (!ed.isOK)
{
return(false);
}
data = Asn1Util.HexStrToBytes(ed.GetValueStr());
aNode.Data = data;
}
break;
}
;
}
return(true);
}
public void SimpleApiServer()
{
long ssn = 555555;
UInt16 age = 1;
VelocityDbSchema.Person mats;
Placement place;
using (ServerClientSession session = new ServerClientSession(systemDir))
{
// skip delete database since it invalidates indices
session.BeginUpdate();
Database db = session.OpenDatabase(10, true, false);
if (db != null)
{
session.DeleteDatabase(db);
}
session.Commit();
session.BeginUpdate();
place = new Placement(10, 2, 1, 1, 10);
DateTime birthday = new DateTime(1960, 6, 13);
mats = new Man("Mats", "Persson", age++, ssn++, birthday);
mats.Persist(place, session);
session.Commit();
session.ClearServerCache();
}
UInt64 mOid1 = Oid.Encode(10, 2, 1);
using (ServerClientSession session = new ServerClientSession(systemDir))
{
session.BeginUpdate();
UInt32 dbNum = Oid.DatabaseNumber(mOid1);
mats = (VelocityDbSchema.Person)session.Open(mOid1);
Woman kinga = null;
mats = new Man("Mats", "Persson", age++, ssn++, new DateTime(1960, 6, 13));
Cat cat = new Cat("Boze", 8);
mats.m_pets.Add(cat);
Bird bird = new Bird("Pippi", 1);
cat.friends.Add(bird);
mats.Persist(place, session);
kinga = new Woman("Kinga", "Persson", age, ssn, mats, mats);
kinga.Persist(place, session);
VelocityDbSchema.Person robin = new VelocityDbSchema.Person("Robin", "Persson", 13, 1, mats, null);
robin.Persist(place, session);
mOid1 = mats.Id;
mats = null;
mats = (VelocityDbSchema.Person)session.Open(mOid1);
session.Commit();
session.ClearServerCache();
}
using (ServerClientSession session = new ServerClientSession(systemDir))
{
session.BeginUpdate();
mats = (VelocityDbSchema.Person)session.Open(mOid1);
session.Commit();
session.ClearServerCache();
}
using (ServerClientSession session = new ServerClientSession(systemDir))
{
session.BeginRead();
ulong mOid2 = mats.Id;
mats = (VelocityDbSchema.Person)session.Open(mOid2);
session.Commit();
session.ClearServerCache();
}
}
public MainWindow()
{
const ushort btreeNodeSize = 5000;
GCSettings.LatencyMode = GCLatencyMode.Batch;// try to keep the WeakIOptimizedPersistableReference objects around longer
dataGridList = new List <DataGrid>();
dataTableList = new List <DataTable>();
InitializeComponent();
session = new SessionNoServer(s_systemDir);
Placement placerIndexRoot = new Placement(IndexRoot.PlaceInDatabase);
session.BeginUpdate();
Console.WriteLine("Running with databases in directory: " + session.SystemDirectory);
File.Copy(s_licenseDbFile, Path.Combine(session.SystemDirectory, "4.odb"), true);
IndexRoot indexRoot;
Database db = session.OpenDatabase(IndexRoot.PlaceInDatabase, false, false);
if (db == null)
{
session.NewDatabase(IndexRoot.PlaceInDatabase, 0, "IndexRoot");
session.NewDatabase(Lexicon.PlaceInDatabase, 0, "Lexicon");
session.NewDatabase(Document.PlaceInDatabase, 0, "Document");
session.NewDatabase(Repository.PlaceInDatabase, 0, "Repository");
session.NewDatabase(DocumentText.PlaceInDatabase, 0, "DocumentText");
session.NewDatabase(Word.PlaceInDatabase, 0, "Word");
indexRoot = new IndexRoot(btreeNodeSize, session);
if (Directory.Exists(s_booksDir))
{
string[] directoryTextFiles = Directory.GetFiles(s_booksDir, "*.txt");
foreach (string fileName in directoryTextFiles)
{
listBoxPagesToAdd.Items.Add(fileName);
}
}
else
{
wordMinCt.Text = 1.ToString();
listBoxPagesToAdd.Items.Add("http://www.VelocityDB.com/");
// other database products
listBoxPagesToAdd.Items.Add("https://foundationdb.com/");
listBoxPagesToAdd.Items.Add("http://www.oracle.com/us/products/database/index.html");
listBoxPagesToAdd.Items.Add("http://www-01.ibm.com/software/data/db2/");
listBoxPagesToAdd.Items.Add("http://www.versant.com/");
listBoxPagesToAdd.Items.Add("http://web.progress.com/en/objectstore/");
listBoxPagesToAdd.Items.Add("https://www.mongodb.org/");
listBoxPagesToAdd.Items.Add("http://cassandra.apache.org/");
listBoxPagesToAdd.Items.Add("http://www.sybase.com/");
listBoxPagesToAdd.Items.Add("http://www.mcobject.com/perst");
listBoxPagesToAdd.Items.Add("http://www.marklogic.com/what-is-marklogic/");
listBoxPagesToAdd.Items.Add("http://hamsterdb.com/");
listBoxPagesToAdd.Items.Add("http://www.firebirdsql.org/");
listBoxPagesToAdd.Items.Add("http://www.h2database.com/");
listBoxPagesToAdd.Items.Add("http://www.oracle.com/technology/products/berkeley-db");
listBoxPagesToAdd.Items.Add("http://www.scimore.com/");
listBoxPagesToAdd.Items.Add("http://www.stsdb.com/");
listBoxPagesToAdd.Items.Add("http://www.sqlite.org/about.html");
listBoxPagesToAdd.Items.Add("http://www.mysql.com/products/enterprise/techspec.html");
listBoxPagesToAdd.Items.Add("http://www.objectivity.com");
listBoxPagesToAdd.Items.Add("http://vistadb.net/");
listBoxPagesToAdd.Items.Add("http://www.google.com/search?q=object+database&sourceid=ie7&rls=com.microsoft:en-us:IE-SearchBox&ie=&oe=");
}
indexRoot.Persist(session, indexRoot);
}
else
{
indexRoot = (IndexRoot)session.Open(Oid.Encode(IndexRoot.PlaceInDatabase, 1, 1));
}
if (indexRoot.repository.documentSet.Count > 0)
{
List <Document> docs = indexRoot.repository.documentSet.ToList <Document>().Take(50).ToList <Document>();
inDbListBox.ItemsSource = docs;
}
updateDataGrids(indexRoot);
session.Commit();
//verify();
}