private static void doSpellCheckerIndexing(string LuceneIndexDir, string SpellCheckerIndexDir) { try { // http://lucene.apache.org/java/2_2_0/api/org/apache/lucene/search/spell/SpellChecker.html FSDirectory spellCheckerIndexDir = FSDirectory.GetDirectory(SpellCheckerIndexDir, false); FSDirectory indexDir = FSDirectory.GetDirectory(LuceneIndexDir, false); SpellChecker.Net.Search.Spell.SpellChecker spellchecker = new SpellChecker.Net.Search.Spell.SpellChecker(spellCheckerIndexDir); spellchecker.ClearIndex(); // SpellChecker.Net.Search.Spell.SpellChecker spellchecker = new SpellChecker.Net.Search.Spell.SpellChecker (global::Lucene.Net.Store.Directory SpellChecker(spellIndexDirectory); IndexReader r = IndexReader.Open(indexDir); try { // To index a field of a user index: Dictionary dict = new SpellChecker.Net.Search.Spell.LuceneDictionary(r, "title"); spellchecker.IndexDictionary(dict); } finally { r.Close(); } } catch (Exception ex) { Console.Write("Could not create spell-checking index" + ex.Message); } }
} // SearchActiveDocument public string getSpellingSuggestion(string query) { FSDirectory indexDir = FSDirectory.GetDirectory(this.spellingIndexDir, false); SpellChecker.Net.Search.Spell.SpellChecker spellchecker = new SpellChecker.Net.Search.Spell.SpellChecker(indexDir); IndexReader my_lucene_reader = IndexReader.Open(indexDir); string[] words = query.Split(new char[] { ' ', ',', ';' }, StringSplitOptions.RemoveEmptyEntries); List <string> allSuggestions = new List <string>(); foreach (string word in words) { string[] suggestions = spellchecker.SuggestSimilar(word, 1); if (suggestions.Length > 0) { allSuggestions.Add(suggestions[0]); } else { allSuggestions.Add(word); } } string completeSuggestion = String.Join(" ", allSuggestions.ToArray()); return(completeSuggestion); }
public virtual void TestFSDirectoryClone() { System.String tempDir = System.IO.Path.GetTempPath(); if (tempDir == null) { throw new System.IO.IOException("java.io.tmpdir undefined, cannot run test"); } System.IO.FileInfo indexDir2 = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "FSDirIndexReaderClone")); Directory dir1 = FSDirectory.GetDirectory(indexDir2); TestIndexReaderReopen.CreateIndex(dir1, false); IndexReader reader = IndexReader.Open(indexDir2); IndexReader readOnlyReader = (IndexReader)reader.Clone(); reader.Close(); readOnlyReader.Close(); // Make sure we didn't pick up too many incRef's along // the way -- this close should be the final close: dir1.Close(); try { dir1.ListAll(); Assert.Fail("did not hit AlreadyClosedException"); } catch (AlreadyClosedException ace) { // expected } }
public static void Main(System.String[] args) { try { Directory directory = FSDirectory.GetDirectory("demo index", false); IndexReader reader = IndexReader.Open(directory); // Term term = new Term("path", "pizza"); // int deleted = reader.delete(term); // System.out.println("deleted " + deleted + // " documents containing " + term); for (int i = 0; i < reader.MaxDoc(); i++) { reader.Delete(i); } reader.Close(); directory.Close(); } catch (System.Exception e) { System.Console.Out.WriteLine(" caught a " + e.GetType() + "\n with message: " + e.Message); } }
/// <summary> Open an index with write access. /// /// </summary> /// <param name="dirName">the index directory /// </param> /// <param name="analyzer">the analyzer to use for adding new documents /// </param> /// <param name="create"><code>true</code> to create the index or overwrite the existing one; /// <code>false</code> to append to the existing index /// </param> public IndexModifier(System.String dirName, Analyzer analyzer, bool create) { InitBlock(); Directory dir = FSDirectory.GetDirectory(dirName); Init(dir, analyzer, create); }
/// <summary> Open an index with write access. /// /// </summary> /// <param name="file">the index directory /// </param> /// <param name="analyzer">the analyzer to use for adding new documents /// </param> /// <param name="create"><code>true</code> to create the index or overwrite the existing one; /// <code>false</code> to append to the existing index /// </param> public IndexModifier(System.IO.FileInfo file, Analyzer analyzer, bool create) { InitBlock(); Directory dir = FSDirectory.GetDirectory(file); Init(dir, analyzer, create); }
public static void Main(System.String[] args) { try { Directory directory = FSDirectory.GetDirectory("test", true); directory.Close(); IndexDoc("one", "test.txt"); PrintSegment("one"); IndexDoc("two", "test2.txt"); PrintSegment("two"); Merge("one", "two", "merge"); PrintSegment("merge"); Merge("one", "two", "merge2"); PrintSegment("merge2"); Merge("merge", "merge2", "merge3"); PrintSegment("merge3"); } catch (System.Exception e) { System.Console.Out.WriteLine(" caught a " + e.GetType() + "\n with message: " + e.Message); System.Console.Error.WriteLine(e.StackTrace); } }
/// <summary> Reads version number from segments files. The version number is /// initialized with a timestamp and then increased by one for each change of /// the index. /// /// </summary> /// <param name="directory">where the index resides. /// </param> /// <returns> version number. /// </returns> /// <throws> IOException if segments file cannot be read </throws> public static long GetCurrentVersion(System.IO.FileInfo directory) { Directory dir = FSDirectory.GetDirectory(directory, false); long version = GetCurrentVersion(dir); dir.Close(); return(version); }
/// <summary> Returns <code>true</code> iff the index in the named directory is /// currently locked. /// </summary> /// <param name="directory">the directory to check for a lock /// </param> /// <throws> IOException if there is a problem with accessing the index </throws> public static bool IsLocked(System.String directory) { Directory dir = FSDirectory.GetDirectory(directory, false); bool result = IsLocked(dir); dir.Close(); return(result); }
protected internal FSDirectory GetDirectory(bool create) { if (directory == null) { directory = FSDirectory.GetDirectory(SupportClass.AppSettings.Get("test.index.dir", "."), create); } return(directory); }
private void IndexDoc(System.String segment, System.String fileName) { Directory directory = FSDirectory.GetDirectory(indexDir, false); Analyzer analyzer = new SimpleAnalyzer(); DocumentWriter writer = new DocumentWriter(directory, analyzer, Similarity.GetDefault(), 1000); System.IO.FileInfo file = new System.IO.FileInfo(workDir.FullName + "\\" + fileName); Document doc = FileDocument.Document(file); writer.AddDocument(segment, doc); directory.Close(); }
public static void IndexDoc(System.String segment, System.String fileName) { Directory directory = FSDirectory.GetDirectory("test", false); Analyzer analyzer = new SimpleAnalyzer(); DocumentWriter writer = new DocumentWriter(directory, analyzer, Similarity.GetDefault(), 1000); System.IO.FileInfo file = new System.IO.FileInfo(fileName); Document doc = Lucene.Net.Demo.FileDocument.Document(file); writer.AddDocument(segment, doc); directory.Close(); }
public virtual void TestIndexAndMerge() { System.IO.StringWriter out_Renamed = new System.IO.StringWriter(); Directory directory = FSDirectory.GetDirectory(indexDir, true); directory.Close(); IndexDoc("one", "test.txt"); PrintSegment(out_Renamed, "one"); IndexDoc("two", "test2.txt"); PrintSegment(out_Renamed, "two"); Merge("one", "two", "merge", false); PrintSegment(out_Renamed, "merge"); Merge("one", "two", "merge2", false); PrintSegment(out_Renamed, "merge2"); Merge("merge", "merge2", "merge3", false); PrintSegment(out_Renamed, "merge3"); out_Renamed.Close(); System.String multiFileOutput = out_Renamed.GetStringBuilder().ToString(); out_Renamed = new System.IO.StringWriter(); directory = FSDirectory.GetDirectory(indexDir, true); directory.Close(); IndexDoc("one", "test.txt"); PrintSegment(out_Renamed, "one"); IndexDoc("two", "test2.txt"); PrintSegment(out_Renamed, "two"); Merge("one", "two", "merge", true); PrintSegment(out_Renamed, "merge"); Merge("one", "two", "merge2", true); PrintSegment(out_Renamed, "merge2"); Merge("merge", "merge2", "merge3", true); PrintSegment(out_Renamed, "merge3"); out_Renamed.Close(); System.String singleFileOutput = out_Renamed.GetStringBuilder().ToString(); Assert.AreEqual(multiFileOutput, singleFileOutput); }
public static void Main(System.String[] args) { bool doFix = false; for (int i = 0; i < args.Length; i++) { if (args[i].Equals("-fix")) { doFix = true; break; } } if (args.Length != (doFix ? 2 : 1)) { out_Renamed.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index. Always make\n" + "a backup copy of your index before running this! Do not run this tool on an index\n" + "that is actively being written to. You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified. With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file. This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has has any\n" + "corruption, else 0.\n"); System.Environment.Exit(1); } System.String dirName = args[0]; out_Renamed.WriteLine("\nOpening index @ " + dirName + "\n"); Directory dir = null; try { dir = FSDirectory.GetDirectory(dirName); } catch (System.Exception t) { out_Renamed.WriteLine("ERROR: could not open directory \"" + dirName + "\"; exiting"); out_Renamed.Write(t.StackTrace); out_Renamed.Flush(); System.Environment.Exit(1); } bool isClean = Check(dir, doFix); int exitCode; if (isClean) { exitCode = 0; } else { exitCode = 1; } System.Environment.Exit(exitCode); }
internal static void Merge(System.String seg1, System.String seg2, System.String merged) { Directory directory = FSDirectory.GetDirectory("test", false); SegmentReader r1 = new SegmentReader(new SegmentInfo(seg1, 1, directory)); SegmentReader r2 = new SegmentReader(new SegmentInfo(seg2, 1, directory)); SegmentMerger merger = new SegmentMerger(directory, merged, false); merger.Add(r1); merger.Add(r2); merger.Merge(); merger.CloseReaders(); directory.Close(); }
private void Merge(System.String seg1, System.String seg2, System.String merged, bool useCompoundFile) { Directory directory = FSDirectory.GetDirectory(indexDir, false); SegmentReader r1 = new SegmentReader(new SegmentInfo(seg1, 1, directory)); SegmentReader r2 = new SegmentReader(new SegmentInfo(seg2, 1, directory)); SegmentMerger merger = new SegmentMerger(directory, merged, useCompoundFile); merger.Add(r1); merger.Add(r2); merger.Merge(); merger.CloseReaders(); directory.Close(); }
public virtual void SetUp() { workDir = new System.IO.FileInfo(System.Configuration.ConfigurationSettings.AppSettings.Get("tempDir") + "\\" + "TestDoc"); System.IO.Directory.CreateDirectory(workDir.FullName); indexDir = new System.IO.FileInfo(workDir.FullName + "\\" + "testIndex"); System.IO.Directory.CreateDirectory(indexDir.FullName); Directory directory = FSDirectory.GetDirectory(indexDir, true); directory.Close(); files = new System.Collections.ArrayList(); files.Add(CreateFile("test.txt", "This is the first test file")); files.Add(CreateFile("test2.txt", "This is the second test file")); }
internal static void PrintSegment(System.String segment) { Directory directory = FSDirectory.GetDirectory("test", false); SegmentReader reader = new SegmentReader(new SegmentInfo(segment, 1, directory)); for (int i = 0; i < reader.NumDocs(); i++) { System.Console.Out.WriteLine(reader.Document(i)); } TermEnum tis = reader.Terms(); while (tis.Next()) { System.Console.Out.Write(tis.Term()); System.Console.Out.WriteLine(" DF=" + tis.DocFreq()); TermPositions positions = reader.TermPositions(tis.Term()); try { while (positions.Next()) { System.Console.Out.Write(" doc=" + positions.Doc()); System.Console.Out.Write(" TF=" + positions.Freq()); System.Console.Out.Write(" pos="); System.Console.Out.Write(positions.NextPosition()); for (int j = 1; j < positions.Freq(); j++) { System.Console.Out.Write("," + positions.NextPosition()); } System.Console.Out.WriteLine(""); } } finally { positions.Close(); } } tis.Close(); reader.Close(); directory.Close(); }
private void PrintSegment(System.IO.StringWriter out_Renamed, System.String segment) { Directory directory = FSDirectory.GetDirectory(indexDir, false); SegmentReader reader = new SegmentReader(new SegmentInfo(segment, 1, directory)); for (int i = 0; i < reader.NumDocs(); i++) { out_Renamed.WriteLine(reader.Document(i)); } TermEnum tis = reader.Terms(); while (tis.Next()) { out_Renamed.Write(tis.Term()); out_Renamed.WriteLine(" DF=" + tis.DocFreq()); TermPositions positions = reader.TermPositions(tis.Term()); try { while (positions.Next()) { out_Renamed.Write(" doc=" + positions.Doc()); out_Renamed.Write(" TF=" + positions.Freq()); out_Renamed.Write(" pos="); out_Renamed.Write(positions.NextPosition()); for (int j = 1; j < positions.Freq(); j++) { out_Renamed.Write("," + positions.NextPosition()); } out_Renamed.WriteLine(""); } } finally { positions.Close(); } } tis.Close(); reader.Close(); directory.Close(); }
/// <summary> Constructs an IndexWriter for the index in <code>path</code>. /// Text will be analyzed with <code>a</code>. If <code>create</code> /// is true, then a new, empty index will be created in /// <code>path</code>, replacing the index already there, if any. /// /// </summary> /// <param name="path">the path to the index directory /// </param> /// <param name="a">the analyzer to use /// </param> /// <param name="create"><code>true</code> to create the index or overwrite /// the existing one; <code>false</code> to append to the existing /// index /// </param> /// <throws> IOException if the directory cannot be read/written to, or </throws> /// <summary> if it does not exist, and <code>create</code> is /// <code>false</code> /// </summary> public IndexWriter(System.IO.FileInfo path, Analyzer a, bool create) : this(FSDirectory.GetDirectory(path, create), a, create, true) { }
public virtual void SetUp() { //dir = new RAMDirectory(); dir = FSDirectory.GetDirectory(new System.IO.FileInfo(SupportClass.AppSettings.Get("tempDir", "testIndex")), true); }
public static void Main(System.String[] args) { System.String filename = null; bool extract = false; for (int i = 0; i < args.Length; ++i) { if (args[i].Equals("-extract")) { extract = true; } else if (filename == null) { filename = args[i]; } } if (filename == null) { System.Console.Out.WriteLine("Usage: Lucene.Net.index.IndexReader [-extract] <cfsfile>"); return; } Directory dir = null; CompoundFileReader cfr = null; try { System.IO.FileInfo file = new System.IO.FileInfo(filename); System.String dirname = new System.IO.FileInfo(file.FullName).DirectoryName; filename = file.Name; dir = FSDirectory.GetDirectory(dirname, false); cfr = new CompoundFileReader(dir, filename); System.String[] files = cfr.List(); System.Array.Sort(files); // sort the array of filename so that the output is more readable for (int i = 0; i < files.Length; ++i) { long len = cfr.FileLength(files[i]); if (extract) { System.Console.Out.WriteLine("extract " + files[i] + " with " + len + " bytes to local directory..."); IndexInput ii = cfr.OpenInput(files[i]); System.IO.FileStream f = new System.IO.FileStream(files[i], System.IO.FileMode.Create); // read and write with a small buffer, which is more effectiv than reading byte by byte byte[] buffer = new byte[1024]; int chunk = buffer.Length; while (len > 0) { int bufLen = (int)System.Math.Min(chunk, len); ii.ReadBytes(buffer, 0, bufLen); byte[] byteArray = new byte[buffer.Length]; for (int index = 0; index < buffer.Length; index++) { byteArray[index] = (byte)buffer[index]; } f.Write(byteArray, 0, bufLen); len -= bufLen; } f.Close(); ii.Close(); } else { System.Console.Out.WriteLine(files[i] + ": " + len + " bytes"); } } } catch (System.IO.IOException ioe) { System.Console.Error.WriteLine(ioe.StackTrace); } finally { try { if (dir != null) { dir.Close(); } if (cfr != null) { cfr.Close(); } } catch (System.IO.IOException ioe) { System.Console.Error.WriteLine(ioe.StackTrace); } } }
public IndexableFileInfo[] getRelatedFiles(string title, int maxResultsToReturn) { // http://blogs.intesoft.net/post/2008/04/NHibernateSearch-using-LuceneNET-Full-Text-Index-(Part-3).aspx Analyzer analyzer = new StandardAnalyzer(); BooleanQuery query = new BooleanQuery(); if (title.Trim() != "") { Query titleQ = Similarity.Net.SimilarityQueries.FormSimilarQuery(title, analyzer, "title", null); titleQ.SetBoost(LuceneIndexer.TitleFieldBoost); query.Add(titleQ, BooleanClause.Occur.SHOULD); Query contents = Similarity.Net.SimilarityQueries.FormSimilarQuery(title, analyzer, "contents", null); query.Add(contents, BooleanClause.Occur.SHOULD); } // avoid the page being similar to itself! // query.Add(new TermQuery(new Term("title", title)), BooleanClause.Occur.MUST_NOT); /// IndexReader ir = ... /// IndexSearcher is = ... /// <b> /// MoreLikeThis mlt = new MoreLikeThis(ir); /// Reader target = ... </b><em>// orig source of doc you want to find similarities to</em><b> /// Query query = mlt.Like( target); /// </b> /// Hits hits = is.Search(query); FSDirectory indexDir = FSDirectory.GetDirectory(this.luceneIndexDir, false); IndexSearcher searcher; try { searcher = new IndexSearcher(indexDir); } catch { // if the luceneIndexDir does not contain index files (yet), IndexSearcher // throws a nice Exception. return(new IndexableFileInfo[0]); } List <IndexableFileInfo> arrayList = new List <IndexableFileInfo>(); Hits hits = searcher.Search(query); try { int num = Math.Min(maxResultsToReturn, hits.Length()); for (int i = 0; i < num; i++) { Document d = hits.Doc(i); string filename = d.Get("filename"); string plainText = d.Get("contents"); string doctitle = d.Get("title"); string filenameParams = d.Get("filenameParams"); bool contentIsPageSummary = Convert.ToBoolean(d.Get("contentIsPageSummary")); DateTime lastModified = DateTools.StringToDate(d.Get("LastModified")); string fragment = plainText; string sectionName = d.Get("SectionName"); IndexableFileInfo newHit = new IndexableFileInfo(filename, filenameParams, doctitle, fragment, sectionName, lastModified, contentIsPageSummary); arrayList.Add(newHit); } // for } finally { searcher.Close(); } return(arrayList.ToArray()); }
} // constructor /// <summary> /// Searches the keyword index using the keywordQuery. /// /// See http://www.dotlucene.net/documentation/QuerySyntax.html for the format of the keywordQuery. /// /// This function will return a fully-filled array of IndexableFileInfo objects. /// </summary> /// <param name="keywordQuery"></param> /// <param name="queryForHighlighter"></param> /// <returns></returns> public IndexableFileInfo[] doSearch(string keywordQuery, string queryForHighlighter) { IndexSearcher searcher; IndexReader indexReader; try { FSDirectory indexDir = FSDirectory.GetDirectory(this.luceneIndexDir, false); indexReader = IndexReader.Open(indexDir); searcher = new IndexSearcher(indexReader); } catch { // if the luceneIndexDir does not contain index files (yet), IndexSearcher // throws a nice Exception. return(new IndexableFileInfo[0]); } List <IndexableFileInfo> arrayList = new List <IndexableFileInfo>(); try { string Query = keywordQuery; if (Query == String.Empty) { return(new IndexableFileInfo[0]); } string HighlighterQuery = queryForHighlighter; // -- weirdly enough, when the query is empty, an exception is thrown during the QueryParser.Parse // this hack gets around that. if (HighlighterQuery == String.Empty) { HighlighterQuery = Guid.NewGuid().ToString(); } // parse the query, "text" is the default field to search // note: use the StandardAnalyzer! (the SimpleAnalyzer doesn't work correctly when searching by fields that are integers!) // MultiFieldQueryParser queryParser = new MultiFieldQueryParser(new string[] { "title", "contents" }, new hatWebPortalAnalyzer()); MultiFieldQueryParser queryParser = new MultiFieldQueryParser(new string[] { "title", "contents" }, new SimpleAnalyzer()); queryParser.SetDefaultOperator(QueryParser.AND_OPERATOR); Query query = queryParser.Parse(Query); QueryParser highlightQueryParser = new QueryParser("contents", new hatWebPortalAnalyzer()); Query highlighterQuery = highlightQueryParser.Parse(HighlighterQuery); query = searcher.Rewrite(query); // is this needed?? " Expert: called to re-write queries into primitive queries." // search Hits hits = searcher.Search(query, Sort.RELEVANCE); // create highlighter Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter("<strong>", "</strong>"), new QueryScorer(highlighterQuery)); // -- go through hits and return results for (int i = 0; i < hits.Length(); i++) { Document d = hits.Doc(i); string filename = d.Get("filename"); string plainText = d.Get("contents"); string title = d.Get("title"); string sectionName = d.Get("SectionName"); string filenameParams = d.Get("filenameParams"); bool contentIsPageSummary = Convert.ToBoolean(d.Get("contentIsPageSummary")); double score = Convert.ToDouble(hits.Score(i)); DateTime lastModified = DateTools.StringToDate(d.Get("LastModified")); TokenStream tokenStream = new hatWebPortalAnalyzer().TokenStream("contents", new StringReader(plainText)); string fragment = plainText; if (!contentIsPageSummary) { fragment = highlighter.GetBestFragments(tokenStream, plainText, 2, "..."); } IndexableFileInfo newHit = new IndexableFileInfo(filename, filenameParams, title, fragment, sectionName, lastModified, contentIsPageSummary, score); arrayList.Add(newHit); } // for } finally { searcher.Close(); indexReader.Close(); } return(arrayList.ToArray()); } // SearchActiveDocument
/// <summary>Returns an IndexReader reading the index in an FSDirectory in the named /// path. /// </summary> public static IndexReader Open(System.IO.FileInfo path) { return(Open(FSDirectory.GetDirectory(path, false), true)); }
/// <summary>Returns an IndexReader reading the index in an FSDirectory in the named /// path. /// </summary> public static IndexReader Open(System.String path) { return(Open(FSDirectory.GetDirectory(path, false), true)); }
private Directory GetDirectory(bool create) { return(FSDirectory.GetDirectory(new System.IO.FileInfo(SupportClass.AppSettings.Get("tempDir", "") + "\\" + "testIndex"), create)); }
public static void Test(int count, bool ram) { System.Random gen = new System.Random((System.Int32) 1251971); int i; System.DateTime veryStart = System.DateTime.Now; System.DateTime start = System.DateTime.Now; Directory store; if (ram) { store = new RAMDirectory(); } else { store = FSDirectory.GetDirectory("test.store", true); } int LENGTH_MASK = 0xFFF; for (i = 0; i < count; i++) { System.String name = i + ".dat"; int length = gen.Next() & LENGTH_MASK; byte b = (byte)(gen.Next() & 0x7F); //System.out.println("filling " + name + " with " + length + " of " + b); OutputStream file = store.CreateFile(name); for (int j = 0; j < length; j++) { file.WriteByte(b); } file.Close(); } store.Close(); System.DateTime end = System.DateTime.Now; System.Console.Out.Write(end.Ticks - start.Ticks); System.Console.Out.WriteLine(" total milliseconds to create"); gen = new System.Random((System.Int32) 1251971); start = System.DateTime.Now; if (!ram) { store = FSDirectory.GetDirectory("test.store", false); } for (i = 0; i < count; i++) { System.String name = i + ".dat"; int length = gen.Next() & LENGTH_MASK; sbyte b = (sbyte)(gen.Next() & 0x7F); //System.out.println("reading " + name + " with " + length + " of " + b); InputStream file = store.OpenFile(name); if (file.Length() != length) { throw new System.Exception("length incorrect"); } for (int j = 0; j < length; j++) { if (file.ReadByte() != b) { throw new System.Exception("contents incorrect"); } } file.Close(); } end = System.DateTime.Now; System.Console.Out.Write(end.Ticks - start.Ticks); System.Console.Out.WriteLine(" total milliseconds to read"); gen = new System.Random((System.Int32) 1251971); start = System.DateTime.Now; for (i = 0; i < count; i++) { System.String name = i + ".dat"; //System.out.println("deleting " + name); store.DeleteFile(name); } end = System.DateTime.Now; System.Console.Out.Write(end.Ticks - start.Ticks); System.Console.Out.WriteLine(" total milliseconds to delete"); System.Console.Out.Write(end.Ticks - veryStart.Ticks); System.Console.Out.WriteLine(" total milliseconds"); store.Close(); }
// FIXME: OG: remove hard-coded file names public static void Test() { System.IO.FileInfo file = new System.IO.FileInfo("words.txt"); System.Console.Out.WriteLine(" reading word file containing " + file.Length + " bytes"); System.DateTime start = System.DateTime.Now; System.Collections.ArrayList keys = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(10)); System.IO.FileStream ws = new System.IO.FileStream(file.FullName, System.IO.FileMode.Open, System.IO.FileAccess.Read); System.IO.StreamReader wr = new System.IO.StreamReader(new System.IO.StreamReader(ws, System.Text.Encoding.Default).BaseStream, new System.IO.StreamReader(ws, System.Text.Encoding.Default).CurrentEncoding); for (System.String key = wr.ReadLine(); key != null; key = wr.ReadLine()) { keys.Add(new Term("word", key)); } wr.Close(); System.DateTime end = System.DateTime.Now; System.Console.Out.Write(end.Ticks - start.Ticks); System.Console.Out.WriteLine(" milliseconds to read " + keys.Count + " words"); start = System.DateTime.Now; System.Random gen = new System.Random((System.Int32) 1251971); long fp = (gen.Next() & 0xF) + 1; long pp = (gen.Next() & 0xF) + 1; int[] docFreqs = new int[keys.Count]; long[] freqPointers = new long[keys.Count]; long[] proxPointers = new long[keys.Count]; for (int i = 0; i < keys.Count; i++) { docFreqs[i] = (gen.Next() & 0xF) + 1; freqPointers[i] = fp; proxPointers[i] = pp; fp += (gen.Next() & 0xF) + 1; ; pp += (gen.Next() & 0xF) + 1; ; } end = System.DateTime.Now; System.Console.Out.Write(end.Ticks - start.Ticks); System.Console.Out.WriteLine(" milliseconds to generate values"); start = System.DateTime.Now; Directory store = FSDirectory.GetDirectory("test.store", true); FieldInfos fis = new FieldInfos(); TermInfosWriter writer = new TermInfosWriter(store, "words", fis); fis.Add("word", false); for (int i = 0; i < keys.Count; i++) { writer.Add((Term)keys[i], new TermInfo(docFreqs[i], freqPointers[i], proxPointers[i])); } writer.Close(); end = System.DateTime.Now; System.Console.Out.Write(end.Ticks - start.Ticks); System.Console.Out.WriteLine(" milliseconds to write table"); System.Console.Out.WriteLine(" table occupies " + store.FileLength("words.tis") + " bytes"); start = System.DateTime.Now; TermInfosReader reader = new TermInfosReader(store, "words", fis); end = System.DateTime.Now; System.Console.Out.Write(end.Ticks - start.Ticks); System.Console.Out.WriteLine(" milliseconds to open table"); start = System.DateTime.Now; SegmentTermEnum enumerator = reader.Terms(); for (int i = 0; i < keys.Count; i++) { enumerator.Next(); Term key = (Term)keys[i]; if (!key.Equals(enumerator.Term())) { throw new System.Exception("wrong term: " + enumerator.Term() + ", expected: " + key + " at " + i); } TermInfo ti = enumerator.TermInfo(); if (ti.docFreq != docFreqs[i]) { throw new System.Exception("wrong value: " + System.Convert.ToString(ti.docFreq, 16) + ", expected: " + System.Convert.ToString(docFreqs[i], 16) + " at " + i); } if (ti.freqPointer != freqPointers[i]) { throw new System.Exception("wrong value: " + System.Convert.ToString(ti.freqPointer, 16) + ", expected: " + System.Convert.ToString(freqPointers[i], 16) + " at " + i); } if (ti.proxPointer != proxPointers[i]) { throw new System.Exception("wrong value: " + System.Convert.ToString(ti.proxPointer, 16) + ", expected: " + System.Convert.ToString(proxPointers[i], 16) + " at " + i); } } end = System.DateTime.Now; System.Console.Out.Write(end.Ticks - start.Ticks); System.Console.Out.WriteLine(" milliseconds to iterate over " + keys.Count + " words"); start = System.DateTime.Now; for (int i = 0; i < keys.Count; i++) { Term key = (Term)keys[i]; TermInfo ti = reader.Get(key); if (ti.docFreq != docFreqs[i]) { throw new System.Exception("wrong value: " + System.Convert.ToString(ti.docFreq, 16) + ", expected: " + System.Convert.ToString(docFreqs[i], 16) + " at " + i); } if (ti.freqPointer != freqPointers[i]) { throw new System.Exception("wrong value: " + System.Convert.ToString(ti.freqPointer, 16) + ", expected: " + System.Convert.ToString(freqPointers[i], 16) + " at " + i); } if (ti.proxPointer != proxPointers[i]) { throw new System.Exception("wrong value: " + System.Convert.ToString(ti.proxPointer, 16) + ", expected: " + System.Convert.ToString(proxPointers[i], 16) + " at " + i); } } end = System.DateTime.Now; System.Console.Out.Write((end.Ticks - start.Ticks) / (float)keys.Count); System.Console.Out.WriteLine(" average milliseconds per lookup"); TermEnum e = reader.Terms(new Term("word", "azz")); System.Console.Out.WriteLine("Word after azz is " + e.Term().text); reader.Close(); store.Close(); }