private static void doSpellCheckerIndexing(string LuceneIndexDir, string SpellCheckerIndexDir)
{
try
{
// http://lucene.apache.org/java/2_2_0/api/org/apache/lucene/search/spell/SpellChecker.html
FSDirectory spellCheckerIndexDir = FSDirectory.GetDirectory(SpellCheckerIndexDir, false);
FSDirectory indexDir = FSDirectory.GetDirectory(LuceneIndexDir, false);
SpellChecker.Net.Search.Spell.SpellChecker spellchecker = new SpellChecker.Net.Search.Spell.SpellChecker(spellCheckerIndexDir);
spellchecker.ClearIndex();
// SpellChecker.Net.Search.Spell.SpellChecker spellchecker = new SpellChecker.Net.Search.Spell.SpellChecker (global::Lucene.Net.Store.Directory SpellChecker(spellIndexDirectory);
IndexReader r = IndexReader.Open(indexDir);
try
{
// To index a field of a user index:
Dictionary dict = new SpellChecker.Net.Search.Spell.LuceneDictionary(r, "title");
spellchecker.IndexDictionary(dict);
}
finally
{
r.Close();
}
}
catch (Exception ex)
{
Console.Write("Could not create spell-checking index" + ex.Message);
}
}
} // SearchActiveDocument
public string getSpellingSuggestion(string query)
{
FSDirectory indexDir = FSDirectory.GetDirectory(this.spellingIndexDir, false);
SpellChecker.Net.Search.Spell.SpellChecker spellchecker = new SpellChecker.Net.Search.Spell.SpellChecker(indexDir);
IndexReader my_lucene_reader = IndexReader.Open(indexDir);
string[] words = query.Split(new char[] { ' ', ',', ';' }, StringSplitOptions.RemoveEmptyEntries);
List <string> allSuggestions = new List <string>();
foreach (string word in words)
{
string[] suggestions = spellchecker.SuggestSimilar(word, 1);
if (suggestions.Length > 0)
{
allSuggestions.Add(suggestions[0]);
}
else
{
allSuggestions.Add(word);
}
}
string completeSuggestion = String.Join(" ", allSuggestions.ToArray());
return(completeSuggestion);
}
public virtual void TestFSDirectoryClone()
{
System.String tempDir = System.IO.Path.GetTempPath();
if (tempDir == null)
{
throw new System.IO.IOException("java.io.tmpdir undefined, cannot run test");
}
System.IO.FileInfo indexDir2 = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "FSDirIndexReaderClone"));
Directory dir1 = FSDirectory.GetDirectory(indexDir2);
TestIndexReaderReopen.CreateIndex(dir1, false);
IndexReader reader = IndexReader.Open(indexDir2);
IndexReader readOnlyReader = (IndexReader)reader.Clone();
reader.Close();
readOnlyReader.Close();
// Make sure we didn't pick up too many incRef's along
// the way -- this close should be the final close:
dir1.Close();
try
{
dir1.ListAll();
Assert.Fail("did not hit AlreadyClosedException");
}
catch (AlreadyClosedException ace)
{
// expected
}
}
public static void Main(System.String[] args)
{
try
{
Directory directory = FSDirectory.GetDirectory("demo index", false);
IndexReader reader = IndexReader.Open(directory);
// Term term = new Term("path", "pizza");
// int deleted = reader.delete(term);
// System.out.println("deleted " + deleted +
// " documents containing " + term);
for (int i = 0; i < reader.MaxDoc(); i++)
{
reader.Delete(i);
}
reader.Close();
directory.Close();
}
catch (System.Exception e)
{
System.Console.Out.WriteLine(" caught a " + e.GetType() + "\n with message: " + e.Message);
}
}
/// <summary> Open an index with write access.
///
/// </summary>
/// <param name="dirName">the index directory
/// </param>
/// <param name="analyzer">the analyzer to use for adding new documents
/// </param>
/// <param name="create"><code>true</code> to create the index or overwrite the existing one;
/// <code>false</code> to append to the existing index
/// </param>
public IndexModifier(System.String dirName, Analyzer analyzer, bool create)
{
InitBlock();
Directory dir = FSDirectory.GetDirectory(dirName);
Init(dir, analyzer, create);
}
/// <summary> Open an index with write access.
///
/// </summary>
/// <param name="file">the index directory
/// </param>
/// <param name="analyzer">the analyzer to use for adding new documents
/// </param>
/// <param name="create"><code>true</code> to create the index or overwrite the existing one;
/// <code>false</code> to append to the existing index
/// </param>
public IndexModifier(System.IO.FileInfo file, Analyzer analyzer, bool create)
{
InitBlock();
Directory dir = FSDirectory.GetDirectory(file);
Init(dir, analyzer, create);
}
public static void Main(System.String[] args)
{
try
{
Directory directory = FSDirectory.GetDirectory("test", true);
directory.Close();
IndexDoc("one", "test.txt");
PrintSegment("one");
IndexDoc("two", "test2.txt");
PrintSegment("two");
Merge("one", "two", "merge");
PrintSegment("merge");
Merge("one", "two", "merge2");
PrintSegment("merge2");
Merge("merge", "merge2", "merge3");
PrintSegment("merge3");
}
catch (System.Exception e)
{
System.Console.Out.WriteLine(" caught a " + e.GetType() + "\n with message: " + e.Message);
System.Console.Error.WriteLine(e.StackTrace);
}
}
/// <summary> Reads version number from segments files. The version number is
/// initialized with a timestamp and then increased by one for each change of
/// the index.
///
/// </summary>
/// <param name="directory">where the index resides.
/// </param>
/// <returns> version number.
/// </returns>
/// <throws> IOException if segments file cannot be read </throws>
public static long GetCurrentVersion(System.IO.FileInfo directory)
{
Directory dir = FSDirectory.GetDirectory(directory, false);
long version = GetCurrentVersion(dir);
dir.Close();
return(version);
}
/// <summary> Returns <code>true</code> iff the index in the named directory is
/// currently locked.
/// </summary>
/// <param name="directory">the directory to check for a lock
/// </param>
/// <throws> IOException if there is a problem with accessing the index </throws>
public static bool IsLocked(System.String directory)
{
Directory dir = FSDirectory.GetDirectory(directory, false);
bool result = IsLocked(dir);
dir.Close();
return(result);
}
protected internal FSDirectory GetDirectory(bool create)
{
if (directory == null)
{
directory = FSDirectory.GetDirectory(SupportClass.AppSettings.Get("test.index.dir", "."), create);
}
return(directory);
}
private void IndexDoc(System.String segment, System.String fileName)
{
Directory directory = FSDirectory.GetDirectory(indexDir, false);
Analyzer analyzer = new SimpleAnalyzer();
DocumentWriter writer = new DocumentWriter(directory, analyzer, Similarity.GetDefault(), 1000);
System.IO.FileInfo file = new System.IO.FileInfo(workDir.FullName + "\\" + fileName);
Document doc = FileDocument.Document(file);
writer.AddDocument(segment, doc);
directory.Close();
}
public static void IndexDoc(System.String segment, System.String fileName)
{
Directory directory = FSDirectory.GetDirectory("test", false);
Analyzer analyzer = new SimpleAnalyzer();
DocumentWriter writer = new DocumentWriter(directory, analyzer, Similarity.GetDefault(), 1000);
System.IO.FileInfo file = new System.IO.FileInfo(fileName);
Document doc = Lucene.Net.Demo.FileDocument.Document(file);
writer.AddDocument(segment, doc);
directory.Close();
}
public virtual void TestIndexAndMerge()
{
System.IO.StringWriter out_Renamed = new System.IO.StringWriter();
Directory directory = FSDirectory.GetDirectory(indexDir, true);
directory.Close();
IndexDoc("one", "test.txt");
PrintSegment(out_Renamed, "one");
IndexDoc("two", "test2.txt");
PrintSegment(out_Renamed, "two");
Merge("one", "two", "merge", false);
PrintSegment(out_Renamed, "merge");
Merge("one", "two", "merge2", false);
PrintSegment(out_Renamed, "merge2");
Merge("merge", "merge2", "merge3", false);
PrintSegment(out_Renamed, "merge3");
out_Renamed.Close();
System.String multiFileOutput = out_Renamed.GetStringBuilder().ToString();
out_Renamed = new System.IO.StringWriter();
directory = FSDirectory.GetDirectory(indexDir, true);
directory.Close();
IndexDoc("one", "test.txt");
PrintSegment(out_Renamed, "one");
IndexDoc("two", "test2.txt");
PrintSegment(out_Renamed, "two");
Merge("one", "two", "merge", true);
PrintSegment(out_Renamed, "merge");
Merge("one", "two", "merge2", true);
PrintSegment(out_Renamed, "merge2");
Merge("merge", "merge2", "merge3", true);
PrintSegment(out_Renamed, "merge3");
out_Renamed.Close();
System.String singleFileOutput = out_Renamed.GetStringBuilder().ToString();
Assert.AreEqual(multiFileOutput, singleFileOutput);
}
public static void Main(System.String[] args)
{
bool doFix = false;
for (int i = 0; i < args.Length; i++)
{
if (args[i].Equals("-fix"))
{
doFix = true;
break;
}
}
if (args.Length != (doFix ? 2 : 1))
{
out_Renamed.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index. Always make\n" + "a backup copy of your index before running this! Do not run this tool on an index\n" + "that is actively being written to. You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified. With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file. This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has has any\n" + "corruption, else 0.\n");
System.Environment.Exit(1);
}
System.String dirName = args[0];
out_Renamed.WriteLine("\nOpening index @ " + dirName + "\n");
Directory dir = null;
try
{
dir = FSDirectory.GetDirectory(dirName);
}
catch (System.Exception t)
{
out_Renamed.WriteLine("ERROR: could not open directory \"" + dirName + "\"; exiting");
out_Renamed.Write(t.StackTrace);
out_Renamed.Flush();
System.Environment.Exit(1);
}
bool isClean = Check(dir, doFix);
int exitCode;
if (isClean)
{
exitCode = 0;
}
else
{
exitCode = 1;
}
System.Environment.Exit(exitCode);
}
internal static void Merge(System.String seg1, System.String seg2, System.String merged)
{
Directory directory = FSDirectory.GetDirectory("test", false);
SegmentReader r1 = new SegmentReader(new SegmentInfo(seg1, 1, directory));
SegmentReader r2 = new SegmentReader(new SegmentInfo(seg2, 1, directory));
SegmentMerger merger = new SegmentMerger(directory, merged, false);
merger.Add(r1);
merger.Add(r2);
merger.Merge();
merger.CloseReaders();
directory.Close();
}
private void Merge(System.String seg1, System.String seg2, System.String merged, bool useCompoundFile)
{
Directory directory = FSDirectory.GetDirectory(indexDir, false);
SegmentReader r1 = new SegmentReader(new SegmentInfo(seg1, 1, directory));
SegmentReader r2 = new SegmentReader(new SegmentInfo(seg2, 1, directory));
SegmentMerger merger = new SegmentMerger(directory, merged, useCompoundFile);
merger.Add(r1);
merger.Add(r2);
merger.Merge();
merger.CloseReaders();
directory.Close();
}
public virtual void SetUp()
{
workDir = new System.IO.FileInfo(System.Configuration.ConfigurationSettings.AppSettings.Get("tempDir") + "\\" + "TestDoc");
System.IO.Directory.CreateDirectory(workDir.FullName);
indexDir = new System.IO.FileInfo(workDir.FullName + "\\" + "testIndex");
System.IO.Directory.CreateDirectory(indexDir.FullName);
Directory directory = FSDirectory.GetDirectory(indexDir, true);
directory.Close();
files = new System.Collections.ArrayList();
files.Add(CreateFile("test.txt", "This is the first test file"));
files.Add(CreateFile("test2.txt", "This is the second test file"));
}
internal static void PrintSegment(System.String segment)
{
Directory directory = FSDirectory.GetDirectory("test", false);
SegmentReader reader = new SegmentReader(new SegmentInfo(segment, 1, directory));
for (int i = 0; i < reader.NumDocs(); i++)
{
System.Console.Out.WriteLine(reader.Document(i));
}
TermEnum tis = reader.Terms();
while (tis.Next())
{
System.Console.Out.Write(tis.Term());
System.Console.Out.WriteLine(" DF=" + tis.DocFreq());
TermPositions positions = reader.TermPositions(tis.Term());
try
{
while (positions.Next())
{
System.Console.Out.Write(" doc=" + positions.Doc());
System.Console.Out.Write(" TF=" + positions.Freq());
System.Console.Out.Write(" pos=");
System.Console.Out.Write(positions.NextPosition());
for (int j = 1; j < positions.Freq(); j++)
{
System.Console.Out.Write("," + positions.NextPosition());
}
System.Console.Out.WriteLine("");
}
}
finally
{
positions.Close();
}
}
tis.Close();
reader.Close();
directory.Close();
}
private void PrintSegment(System.IO.StringWriter out_Renamed, System.String segment)
{
Directory directory = FSDirectory.GetDirectory(indexDir, false);
SegmentReader reader = new SegmentReader(new SegmentInfo(segment, 1, directory));
for (int i = 0; i < reader.NumDocs(); i++)
{
out_Renamed.WriteLine(reader.Document(i));
}
TermEnum tis = reader.Terms();
while (tis.Next())
{
out_Renamed.Write(tis.Term());
out_Renamed.WriteLine(" DF=" + tis.DocFreq());
TermPositions positions = reader.TermPositions(tis.Term());
try
{
while (positions.Next())
{
out_Renamed.Write(" doc=" + positions.Doc());
out_Renamed.Write(" TF=" + positions.Freq());
out_Renamed.Write(" pos=");
out_Renamed.Write(positions.NextPosition());
for (int j = 1; j < positions.Freq(); j++)
{
out_Renamed.Write("," + positions.NextPosition());
}
out_Renamed.WriteLine("");
}
}
finally
{
positions.Close();
}
}
tis.Close();
reader.Close();
directory.Close();
}
/// <summary> Constructs an IndexWriter for the index in <code>path</code>.
/// Text will be analyzed with <code>a</code>. If <code>create</code>
/// is true, then a new, empty index will be created in
/// <code>path</code>, replacing the index already there, if any.
///
/// </summary>
/// <param name="path">the path to the index directory
/// </param>
/// <param name="a">the analyzer to use
/// </param>
/// <param name="create"><code>true</code> to create the index or overwrite
/// the existing one; <code>false</code> to append to the existing
/// index
/// </param>
/// <throws> IOException if the directory cannot be read/written to, or </throws>
/// <summary> if it does not exist, and <code>create</code> is
/// <code>false</code>
/// </summary>
public IndexWriter(System.IO.FileInfo path, Analyzer a, bool create) : this(FSDirectory.GetDirectory(path, create), a, create, true)
{
}
public virtual void SetUp()
{
//dir = new RAMDirectory();
dir = FSDirectory.GetDirectory(new System.IO.FileInfo(SupportClass.AppSettings.Get("tempDir", "testIndex")), true);
}
public static void Main(System.String[] args)
{
System.String filename = null;
bool extract = false;
for (int i = 0; i < args.Length; ++i)
{
if (args[i].Equals("-extract"))
{
extract = true;
}
else if (filename == null)
{
filename = args[i];
}
}
if (filename == null)
{
System.Console.Out.WriteLine("Usage: Lucene.Net.index.IndexReader [-extract] <cfsfile>");
return;
}
Directory dir = null;
CompoundFileReader cfr = null;
try
{
System.IO.FileInfo file = new System.IO.FileInfo(filename);
System.String dirname = new System.IO.FileInfo(file.FullName).DirectoryName;
filename = file.Name;
dir = FSDirectory.GetDirectory(dirname, false);
cfr = new CompoundFileReader(dir, filename);
System.String[] files = cfr.List();
System.Array.Sort(files); // sort the array of filename so that the output is more readable
for (int i = 0; i < files.Length; ++i)
{
long len = cfr.FileLength(files[i]);
if (extract)
{
System.Console.Out.WriteLine("extract " + files[i] + " with " + len + " bytes to local directory...");
IndexInput ii = cfr.OpenInput(files[i]);
System.IO.FileStream f = new System.IO.FileStream(files[i], System.IO.FileMode.Create);
// read and write with a small buffer, which is more effectiv than reading byte by byte
byte[] buffer = new byte[1024];
int chunk = buffer.Length;
while (len > 0)
{
int bufLen = (int)System.Math.Min(chunk, len);
ii.ReadBytes(buffer, 0, bufLen);
byte[] byteArray = new byte[buffer.Length];
for (int index = 0; index < buffer.Length; index++)
{
byteArray[index] = (byte)buffer[index];
}
f.Write(byteArray, 0, bufLen);
len -= bufLen;
}
f.Close();
ii.Close();
}
else
{
System.Console.Out.WriteLine(files[i] + ": " + len + " bytes");
}
}
}
catch (System.IO.IOException ioe)
{
System.Console.Error.WriteLine(ioe.StackTrace);
}
finally
{
try
{
if (dir != null)
{
dir.Close();
}
if (cfr != null)
{
cfr.Close();
}
}
catch (System.IO.IOException ioe)
{
System.Console.Error.WriteLine(ioe.StackTrace);
}
}
}
public IndexableFileInfo[] getRelatedFiles(string title, int maxResultsToReturn)
{
// http://blogs.intesoft.net/post/2008/04/NHibernateSearch-using-LuceneNET-Full-Text-Index-(Part-3).aspx
Analyzer analyzer = new StandardAnalyzer();
BooleanQuery query = new BooleanQuery();
if (title.Trim() != "")
{
Query titleQ = Similarity.Net.SimilarityQueries.FormSimilarQuery(title, analyzer, "title", null);
titleQ.SetBoost(LuceneIndexer.TitleFieldBoost);
query.Add(titleQ, BooleanClause.Occur.SHOULD);
Query contents = Similarity.Net.SimilarityQueries.FormSimilarQuery(title, analyzer, "contents", null);
query.Add(contents, BooleanClause.Occur.SHOULD);
}
// avoid the page being similar to itself!
// query.Add(new TermQuery(new Term("title", title)), BooleanClause.Occur.MUST_NOT);
/// IndexReader ir = ...
/// IndexSearcher is = ...
/// <b>
/// MoreLikeThis mlt = new MoreLikeThis(ir);
/// Reader target = ... </b><em>// orig source of doc you want to find similarities to</em><b>
/// Query query = mlt.Like( target);
/// </b>
/// Hits hits = is.Search(query);
FSDirectory indexDir = FSDirectory.GetDirectory(this.luceneIndexDir, false);
IndexSearcher searcher;
try
{
searcher = new IndexSearcher(indexDir);
}
catch
{
// if the luceneIndexDir does not contain index files (yet), IndexSearcher
// throws a nice Exception.
return(new IndexableFileInfo[0]);
}
List <IndexableFileInfo> arrayList = new List <IndexableFileInfo>();
Hits hits = searcher.Search(query);
try
{
int num = Math.Min(maxResultsToReturn, hits.Length());
for (int i = 0; i < num; i++)
{
Document d = hits.Doc(i);
string filename = d.Get("filename");
string plainText = d.Get("contents");
string doctitle = d.Get("title");
string filenameParams = d.Get("filenameParams");
bool contentIsPageSummary = Convert.ToBoolean(d.Get("contentIsPageSummary"));
DateTime lastModified = DateTools.StringToDate(d.Get("LastModified"));
string fragment = plainText;
string sectionName = d.Get("SectionName");
IndexableFileInfo newHit = new IndexableFileInfo(filename, filenameParams, doctitle, fragment, sectionName, lastModified, contentIsPageSummary);
arrayList.Add(newHit);
} // for
}
finally
{
searcher.Close();
}
return(arrayList.ToArray());
}
} // constructor
/// <summary>
/// Searches the keyword index using the keywordQuery.
///
/// See http://www.dotlucene.net/documentation/QuerySyntax.html for the format of the keywordQuery.
///
/// This function will return a fully-filled array of IndexableFileInfo objects.
/// </summary>
/// <param name="keywordQuery"></param>
/// <param name="queryForHighlighter"></param>
/// <returns></returns>
public IndexableFileInfo[] doSearch(string keywordQuery, string queryForHighlighter)
{
IndexSearcher searcher;
IndexReader indexReader;
try
{
FSDirectory indexDir = FSDirectory.GetDirectory(this.luceneIndexDir, false);
indexReader = IndexReader.Open(indexDir);
searcher = new IndexSearcher(indexReader);
}
catch
{
// if the luceneIndexDir does not contain index files (yet), IndexSearcher
// throws a nice Exception.
return(new IndexableFileInfo[0]);
}
List <IndexableFileInfo> arrayList = new List <IndexableFileInfo>();
try
{
string Query = keywordQuery;
if (Query == String.Empty)
{
return(new IndexableFileInfo[0]);
}
string HighlighterQuery = queryForHighlighter;
// -- weirdly enough, when the query is empty, an exception is thrown during the QueryParser.Parse
// this hack gets around that.
if (HighlighterQuery == String.Empty)
{
HighlighterQuery = Guid.NewGuid().ToString();
}
// parse the query, "text" is the default field to search
// note: use the StandardAnalyzer! (the SimpleAnalyzer doesn't work correctly when searching by fields that are integers!)
// MultiFieldQueryParser queryParser = new MultiFieldQueryParser(new string[] { "title", "contents" }, new hatWebPortalAnalyzer());
MultiFieldQueryParser queryParser = new MultiFieldQueryParser(new string[] { "title", "contents" }, new SimpleAnalyzer());
queryParser.SetDefaultOperator(QueryParser.AND_OPERATOR);
Query query = queryParser.Parse(Query);
QueryParser highlightQueryParser = new QueryParser("contents", new hatWebPortalAnalyzer());
Query highlighterQuery = highlightQueryParser.Parse(HighlighterQuery);
query = searcher.Rewrite(query); // is this needed?? " Expert: called to re-write queries into primitive queries."
// search
Hits hits = searcher.Search(query, Sort.RELEVANCE);
// create highlighter
Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter("<strong>", "</strong>"), new QueryScorer(highlighterQuery));
// -- go through hits and return results
for (int i = 0; i < hits.Length(); i++)
{
Document d = hits.Doc(i);
string filename = d.Get("filename");
string plainText = d.Get("contents");
string title = d.Get("title");
string sectionName = d.Get("SectionName");
string filenameParams = d.Get("filenameParams");
bool contentIsPageSummary = Convert.ToBoolean(d.Get("contentIsPageSummary"));
double score = Convert.ToDouble(hits.Score(i));
DateTime lastModified = DateTools.StringToDate(d.Get("LastModified"));
TokenStream tokenStream = new hatWebPortalAnalyzer().TokenStream("contents", new StringReader(plainText));
string fragment = plainText;
if (!contentIsPageSummary)
{
fragment = highlighter.GetBestFragments(tokenStream, plainText, 2, "...");
}
IndexableFileInfo newHit = new IndexableFileInfo(filename, filenameParams, title, fragment, sectionName, lastModified, contentIsPageSummary, score);
arrayList.Add(newHit);
} // for
}
finally
{
searcher.Close();
indexReader.Close();
}
return(arrayList.ToArray());
} // SearchActiveDocument
/// <summary>Returns an IndexReader reading the index in an FSDirectory in the named
/// path.
/// </summary>
public static IndexReader Open(System.IO.FileInfo path)
{
return(Open(FSDirectory.GetDirectory(path, false), true));
}
/// <summary>Returns an IndexReader reading the index in an FSDirectory in the named
/// path.
/// </summary>
public static IndexReader Open(System.String path)
{
return(Open(FSDirectory.GetDirectory(path, false), true));
}
private Directory GetDirectory(bool create)
{
return(FSDirectory.GetDirectory(new System.IO.FileInfo(SupportClass.AppSettings.Get("tempDir", "") + "\\" + "testIndex"), create));
}
public static void Test(int count, bool ram)
{
System.Random gen = new System.Random((System.Int32) 1251971);
int i;
System.DateTime veryStart = System.DateTime.Now;
System.DateTime start = System.DateTime.Now;
Directory store;
if (ram)
{
store = new RAMDirectory();
}
else
{
store = FSDirectory.GetDirectory("test.store", true);
}
int LENGTH_MASK = 0xFFF;
for (i = 0; i < count; i++)
{
System.String name = i + ".dat";
int length = gen.Next() & LENGTH_MASK;
byte b = (byte)(gen.Next() & 0x7F);
//System.out.println("filling " + name + " with " + length + " of " + b);
OutputStream file = store.CreateFile(name);
for (int j = 0; j < length; j++)
{
file.WriteByte(b);
}
file.Close();
}
store.Close();
System.DateTime end = System.DateTime.Now;
System.Console.Out.Write(end.Ticks - start.Ticks);
System.Console.Out.WriteLine(" total milliseconds to create");
gen = new System.Random((System.Int32) 1251971);
start = System.DateTime.Now;
if (!ram)
{
store = FSDirectory.GetDirectory("test.store", false);
}
for (i = 0; i < count; i++)
{
System.String name = i + ".dat";
int length = gen.Next() & LENGTH_MASK;
sbyte b = (sbyte)(gen.Next() & 0x7F);
//System.out.println("reading " + name + " with " + length + " of " + b);
InputStream file = store.OpenFile(name);
if (file.Length() != length)
{
throw new System.Exception("length incorrect");
}
for (int j = 0; j < length; j++)
{
if (file.ReadByte() != b)
{
throw new System.Exception("contents incorrect");
}
}
file.Close();
}
end = System.DateTime.Now;
System.Console.Out.Write(end.Ticks - start.Ticks);
System.Console.Out.WriteLine(" total milliseconds to read");
gen = new System.Random((System.Int32) 1251971);
start = System.DateTime.Now;
for (i = 0; i < count; i++)
{
System.String name = i + ".dat";
//System.out.println("deleting " + name);
store.DeleteFile(name);
}
end = System.DateTime.Now;
System.Console.Out.Write(end.Ticks - start.Ticks);
System.Console.Out.WriteLine(" total milliseconds to delete");
System.Console.Out.Write(end.Ticks - veryStart.Ticks);
System.Console.Out.WriteLine(" total milliseconds");
store.Close();
}
// FIXME: OG: remove hard-coded file names
public static void Test()
{
System.IO.FileInfo file = new System.IO.FileInfo("words.txt");
System.Console.Out.WriteLine(" reading word file containing " + file.Length + " bytes");
System.DateTime start = System.DateTime.Now;
System.Collections.ArrayList keys = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(10));
System.IO.FileStream ws = new System.IO.FileStream(file.FullName, System.IO.FileMode.Open, System.IO.FileAccess.Read);
System.IO.StreamReader wr = new System.IO.StreamReader(new System.IO.StreamReader(ws, System.Text.Encoding.Default).BaseStream, new System.IO.StreamReader(ws, System.Text.Encoding.Default).CurrentEncoding);
for (System.String key = wr.ReadLine(); key != null; key = wr.ReadLine())
{
keys.Add(new Term("word", key));
}
wr.Close();
System.DateTime end = System.DateTime.Now;
System.Console.Out.Write(end.Ticks - start.Ticks);
System.Console.Out.WriteLine(" milliseconds to read " + keys.Count + " words");
start = System.DateTime.Now;
System.Random gen = new System.Random((System.Int32) 1251971);
long fp = (gen.Next() & 0xF) + 1;
long pp = (gen.Next() & 0xF) + 1;
int[] docFreqs = new int[keys.Count];
long[] freqPointers = new long[keys.Count];
long[] proxPointers = new long[keys.Count];
for (int i = 0; i < keys.Count; i++)
{
docFreqs[i] = (gen.Next() & 0xF) + 1;
freqPointers[i] = fp;
proxPointers[i] = pp;
fp += (gen.Next() & 0xF) + 1;
;
pp += (gen.Next() & 0xF) + 1;
;
}
end = System.DateTime.Now;
System.Console.Out.Write(end.Ticks - start.Ticks);
System.Console.Out.WriteLine(" milliseconds to generate values");
start = System.DateTime.Now;
Directory store = FSDirectory.GetDirectory("test.store", true);
FieldInfos fis = new FieldInfos();
TermInfosWriter writer = new TermInfosWriter(store, "words", fis);
fis.Add("word", false);
for (int i = 0; i < keys.Count; i++)
{
writer.Add((Term)keys[i], new TermInfo(docFreqs[i], freqPointers[i], proxPointers[i]));
}
writer.Close();
end = System.DateTime.Now;
System.Console.Out.Write(end.Ticks - start.Ticks);
System.Console.Out.WriteLine(" milliseconds to write table");
System.Console.Out.WriteLine(" table occupies " + store.FileLength("words.tis") + " bytes");
start = System.DateTime.Now;
TermInfosReader reader = new TermInfosReader(store, "words", fis);
end = System.DateTime.Now;
System.Console.Out.Write(end.Ticks - start.Ticks);
System.Console.Out.WriteLine(" milliseconds to open table");
start = System.DateTime.Now;
SegmentTermEnum enumerator = reader.Terms();
for (int i = 0; i < keys.Count; i++)
{
enumerator.Next();
Term key = (Term)keys[i];
if (!key.Equals(enumerator.Term()))
{
throw new System.Exception("wrong term: " + enumerator.Term() + ", expected: " + key + " at " + i);
}
TermInfo ti = enumerator.TermInfo();
if (ti.docFreq != docFreqs[i])
{
throw new System.Exception("wrong value: " + System.Convert.ToString(ti.docFreq, 16) + ", expected: " + System.Convert.ToString(docFreqs[i], 16) + " at " + i);
}
if (ti.freqPointer != freqPointers[i])
{
throw new System.Exception("wrong value: " + System.Convert.ToString(ti.freqPointer, 16) + ", expected: " + System.Convert.ToString(freqPointers[i], 16) + " at " + i);
}
if (ti.proxPointer != proxPointers[i])
{
throw new System.Exception("wrong value: " + System.Convert.ToString(ti.proxPointer, 16) + ", expected: " + System.Convert.ToString(proxPointers[i], 16) + " at " + i);
}
}
end = System.DateTime.Now;
System.Console.Out.Write(end.Ticks - start.Ticks);
System.Console.Out.WriteLine(" milliseconds to iterate over " + keys.Count + " words");
start = System.DateTime.Now;
for (int i = 0; i < keys.Count; i++)
{
Term key = (Term)keys[i];
TermInfo ti = reader.Get(key);
if (ti.docFreq != docFreqs[i])
{
throw new System.Exception("wrong value: " + System.Convert.ToString(ti.docFreq, 16) + ", expected: " + System.Convert.ToString(docFreqs[i], 16) + " at " + i);
}
if (ti.freqPointer != freqPointers[i])
{
throw new System.Exception("wrong value: " + System.Convert.ToString(ti.freqPointer, 16) + ", expected: " + System.Convert.ToString(freqPointers[i], 16) + " at " + i);
}
if (ti.proxPointer != proxPointers[i])
{
throw new System.Exception("wrong value: " + System.Convert.ToString(ti.proxPointer, 16) + ", expected: " + System.Convert.ToString(proxPointers[i], 16) + " at " + i);
}
}
end = System.DateTime.Now;
System.Console.Out.Write((end.Ticks - start.Ticks) / (float)keys.Count);
System.Console.Out.WriteLine(" average milliseconds per lookup");
TermEnum e = reader.Terms(new Term("word", "azz"));
System.Console.Out.WriteLine("Word after azz is " + e.Term().text);
reader.Close();
store.Close();
}
