private void doTestIndexProperties(bool setIndexProps,
bool indexPropsVal, int numExpectedResults)
{
Dictionary <string, string> props = new Dictionary <string, string>();
// Indexing configuration.
props["analyzer"] = typeof(WhitespaceAnalyzer).AssemblyQualifiedName;
props["content.source"] = typeof(OneDocSource).AssemblyQualifiedName;
props["directory"] = "RAMDirectory";
if (setIndexProps)
{
props["doc.index.props"] = indexPropsVal.ToString();
}
// Create PerfRunData
Config config = new Config(props);
PerfRunData runData = new PerfRunData(config);
TaskSequence tasks = new TaskSequence(runData, TestName, null, false);
tasks.AddTask(new CreateIndexTask(runData));
tasks.AddTask(new AddDocTask(runData));
tasks.AddTask(new CloseIndexTask(runData));
tasks.DoLogic();
IndexReader reader = DirectoryReader.Open(runData.Directory);
IndexSearcher searcher = NewSearcher(reader);
TopDocs td = searcher.Search(new TermQuery(new Term("key", "value")), 10);
assertEquals(numExpectedResults, td.TotalHits);
reader.Dispose();
}
public static IndexWriter ConfigureWriter(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit)
{
IndexWriterConfig iwc = CreateWriterConfig(config, runData, mode, commit);
string infoStreamVal = config.Get("writer.info.stream", null);
if (infoStreamVal != null)
{
if (infoStreamVal.Equals("SystemOut", StringComparison.Ordinal))
{
iwc.SetInfoStream(Console.Out);
}
else if (infoStreamVal.Equals("SystemErr", StringComparison.Ordinal))
{
iwc.SetInfoStream(Console.Error);
}
else
{
FileInfo f = new FileInfo(infoStreamVal);
iwc.SetInfoStream(new StreamWriter(new FileStream(f.FullName, FileMode.Create, FileAccess.Write), Encoding.GetEncoding(0)));
}
}
IndexWriter writer = new IndexWriter(runData.Directory, iwc);
return(writer);
}
public override int DoLogic()
{
PerfRunData runData = RunData;
// Get initial reader
IndexWriter w = runData.IndexWriter;
if (w == null)
{
throw new Exception("please open the writer before invoking NearRealtimeReader");
}
if (runData.GetIndexReader() != null)
{
throw new Exception("please close the existing reader before invoking NearRealtimeReader");
}
long t = J2N.Time.CurrentTimeMilliseconds();
DirectoryReader r = DirectoryReader.Open(w, true);
runData.SetIndexReader(r);
// Transfer our reference to runData
r.DecRef();
// TODO: gather basic metrics for reporting -- eg mean,
// stddev, min/max reopen latencies
// Parent sequence sets stopNow
reopenCount = 0;
while (!Stop)
{
long waitForMsec = (pauseMSec - (J2N.Time.CurrentTimeMilliseconds() - t));
if (waitForMsec > 0)
{
Thread.Sleep((int)waitForMsec);
//System.out.println("NRT wait: " + waitForMsec + " msec");
}
t = J2N.Time.CurrentTimeMilliseconds();
DirectoryReader newReader = DirectoryReader.OpenIfChanged(r);
if (newReader != null)
{
int delay = (int)(J2N.Time.CurrentTimeMilliseconds() - t);
if (reopenTimes.Length == reopenCount)
{
reopenTimes = ArrayUtil.Grow(reopenTimes, 1 + reopenCount);
}
reopenTimes[reopenCount++] = delay;
// TODO: somehow we need to enable warming, here
runData.SetIndexReader(newReader);
// Transfer our reference to runData
newReader.DecRef();
r = newReader;
}
}
Stop = false;
return(reopenCount);
}
public PerfTask(PerfRunData runData)
: this()
{
this.runData = runData;
Config config = runData.Config;
this.maxDepthLogStart = config.Get("task.max.depth.log", 0);
string logStepAtt = "log.step";
string taskLogStepAtt = "log.step." + name;
if (config.Get(taskLogStepAtt, null) != null)
{
logStepAtt = taskLogStepAtt;
}
// It's important to read this from Config, to support vals-by-round.
m_logStep = config.Get(logStepAtt, DEFAULT_LOG_STEP);
// To avoid the check 'if (logStep > 0)' in tearDown(). This effectively
// turns logging off.
if (m_logStep <= 0)
{
m_logStep = int.MaxValue;
}
}
public override int DoLogic()
{
PerfRunData runData = RunData;
runData.TaxonomyWriter = new DirectoryTaxonomyWriter(runData.TaxonomyDir);
return(1);
}
private void doLogStepTest(bool setLogStep, int logStepVal,
bool setTaskLogStep, int taskLogStepVal, int expLogStepValue)
{
PerfRunData runData = createPerfRunData(setLogStep, logStepVal, setTaskLogStep, taskLogStepVal);
MyPerfTask mpt = new MyPerfTask(runData);
assertEquals(expLogStepValue, mpt.getLogStep());
}
public WriteEnwikiLineDocTask(PerfRunData runData)
: base(runData)
{
Stream @out = StreamUtils.GetOutputStream(CategoriesLineFile(new FileInfo(m_fname)));
categoryLineFileOut = new StreamWriter(@out, Encoding.UTF8);
WriteHeader(categoryLineFileOut);
}
public void TestNoParams()
{
PerfRunData runData = createPerfRunData();
new CreateIndexTask(runData).DoLogic();
new CommitIndexTask(runData).DoLogic();
new CloseIndexTask(runData).DoLogic();
}
public void TestNoMergeScheduler()
{
PerfRunData runData = createPerfRunData(null);
runData.Config.Set("merge.scheduler", typeof(NoMergeScheduler).AssemblyQualifiedName);
new CreateIndexTask(runData).DoLogic();
new CloseIndexTask(runData).DoLogic();
}
public void TestNoDeletionPolicy()
{
PerfRunData runData = createPerfRunData(null);
runData.Config.Set("deletion.policy", typeof(NoDeletionPolicy).AssemblyQualifiedName);
new CreateIndexTask(runData).DoLogic();
new CloseIndexTask(runData).DoLogic();
}
public override int DoLogic()
{
PerfRunData runData = RunData;
Config config = runData.Config;
runData.IndexWriter = ConfigureWriter(config, runData, OpenMode.CREATE, null);
return(1);
}
public void TestInfoStream_File()
{
FileInfo outFile = new FileInfo(Path.Combine(getWorkDir().FullName, "infoStreamTest"));
PerfRunData runData = createPerfRunData(outFile.FullName);
new CreateIndexTask(runData).DoLogic();
new CloseIndexTask(runData).DoLogic();
assertTrue(new FileInfo(outFile.FullName).Length > 0);
}
public override void Setup()
{
base.Setup();
//check to make sure either the doc is being stored
PerfRunData runData = RunData;
Config config = runData.Config;
m_clnName = config.Get("collector.class", "");
}
public override int DoLogic()
{
PerfRunData runData = RunData;
DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(runData.TaxonomyDir);
runData.SetTaxonomyReader(taxoReader);
// We transfer reference to the run data
taxoReader.DecRef();
return(1);
}
private readonly object lineFileLock = new object(); // LUCENENET specific - lock to ensure writes don't collide for this instance
public WriteLineDocTask(PerfRunData runData)
: base(runData)
{
Config config = runData.Config;
m_fname = config.Get("line.file.out", null);
if (m_fname == null)
{
throw new ArgumentException("line.file.out must be set");
}
Stream @out = StreamUtils.GetOutputStream(new FileInfo(m_fname));
lineFileOut = new StreamWriter(@out, Encoding.UTF8);
docMaker = runData.DocMaker;
// init fields
string f2r = config.Get("line.fields", null);
if (f2r == null)
{
fieldsToWrite = DEFAULT_FIELDS;
}
else
{
if (f2r.IndexOf(SEP) >= 0)
{
throw new ArgumentException("line.fields " + f2r + " should not contain the separator char: " + SEP);
}
fieldsToWrite = f2r.Split(',').TrimEnd();
}
// init sufficient fields
sufficientFields = new bool[fieldsToWrite.Length];
string suff = config.Get("sufficient.fields", DEFAULT_SUFFICIENT_FIELDS);
if (",".Equals(suff, StringComparison.Ordinal))
{
checkSufficientFields = false;
}
else
{
checkSufficientFields = true;
ISet <string> sf = new JCG.HashSet <string>(suff.Split(',').TrimEnd());
for (int i = 0; i < fieldsToWrite.Length; i++)
{
if (sf.Contains(fieldsToWrite[i]))
{
sufficientFields[i] = true;
}
}
}
WriteHeader(lineFileOut);
}
protected ReadTask(PerfRunData runData) // LUCENENET: CA1012: Abstract types should not have constructors (marked protected)
: base(runData)
{
if (WithSearch)
{
queryMaker = GetQueryMaker();
}
else
{
queryMaker = null;
}
}
public ReadTask(PerfRunData runData)
: base(runData)
{
if (WithSearch)
{
queryMaker = GetQueryMaker();
}
else
{
queryMaker = null;
}
}
public void TestEmptyTitle()
{
FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
PerfRunData runData = createPerfRunData(file, false, typeof(NoTitleDocMaker).AssemblyQualifiedName);
WriteLineDocTask wldt = new WriteLineDocTask(runData);
wldt.DoLogic();
wldt.Dispose();
doReadTest(file, FileType.PLAIN, "", "date", "body");
}
public override void Setup()
{
base.Setup();
//check to make sure either the doc is being stored
PerfRunData data = RunData;
if (data.Config.Get("doc.stored", false) == false)
{
throw new Exception("doc.stored must be set to true");
}
m_maxDocCharsToAnalyze = data.Config.Get("highlighter.maxDocCharsToAnalyze", Highlighter.DEFAULT_MAX_CHARS_TO_ANALYZE);
}
public TaskSequence(PerfRunData runData, String name, TaskSequence parent, bool parallel)
: base(runData)
{
collapsable = (name == null);
name = (name != null ? name : (parallel ? "Par" : "Seq"));
SetName(name);
SetSequenceName();
this.parent = parent;
this.parallel = parallel;
tasks = new List <PerfTask>();
logByTimeMsec = runData.Config.Get("report.time.step.msec", 0);
}
public void TestRegularFile()
{
// Create a document in regular format.
FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
PerfRunData runData = createPerfRunData(file, false, typeof(WriteLineDocMaker).AssemblyQualifiedName);
WriteLineDocTask wldt = new WriteLineDocTask(runData);
wldt.DoLogic();
wldt.Dispose();
doReadTest(file, FileType.PLAIN, "title", "date", "body");
}
public void TestMultiThreaded()
{
FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
PerfRunData runData = createPerfRunData(file, false, typeof(ThreadingDocMaker).AssemblyQualifiedName);
ThreadJob[] threads = new ThreadJob[10];
using (WriteLineDocTask wldt = new WriteLineDocTask(runData))
{
for (int i = 0; i < threads.Length; i++)
{
threads[i] = new ThreadAnonymousHelper("t" + i, wldt);
}
foreach (ThreadJob t in threads)
{
t.Start();
}
foreach (ThreadJob t in threads)
{
t.Join();
}
} // wldt.Dispose();
ISet <String> ids = new JCG.HashSet <string>();
TextReader br = new StreamReader(new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.None), Encoding.UTF8);
try
{
String line = br.ReadLine();
assertHeaderLine(line); // header line is written once, no matter how many threads there are
for (int i = 0; i < threads.Length; i++)
{
line = br.ReadLine();
assertNotNull($"line for index {i} is missing", line); // LUCENENET specific - ensure the line is there before splitting
String[] parts = line.Split(WriteLineDocTask.SEP).TrimEnd();
assertEquals(line, 3, parts.Length);
// check that all thread names written are the same in the same line
String tname = parts[0].Substring(parts[0].IndexOf('_'));
ids.add(tname);
assertEquals(tname, parts[1].Substring(parts[1].IndexOf('_')));
assertEquals(tname, parts[2].Substring(parts[2].IndexOf('_')));
}
// only threads.length lines should exist
assertNull(br.ReadLine());
assertEquals(threads.Length, ids.size());
}
finally
{
br.Dispose();
}
}
public void TestCommitData()
{
PerfRunData runData = createPerfRunData();
new CreateIndexTask(runData).DoLogic();
CommitIndexTask task = new CommitIndexTask(runData);
task.SetParams("params");
task.DoLogic();
SegmentInfos infos = new SegmentInfos();
infos.Read(runData.Directory);
assertEquals("params", infos.UserData[OpenReaderTask.USER_DATA]);
new CloseIndexTask(runData).DoLogic();
}
private void assertIndex(PerfRunData runData)
{
Store.Directory taskDir = runData.Directory;
assertSame(typeof(RAMDirectory), taskDir.GetType());
IndexReader r = DirectoryReader.Open(taskDir);
try
{
assertEquals(10, r.NumDocs);
}
finally
{
r.Dispose();
}
}
public void TestEmptyBody()
{
// WriteLineDocTask threw away documents w/ no BODY element, even if they
// had a TITLE element (LUCENE-1755). It should throw away documents if they
// don't have BODY nor TITLE
FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
PerfRunData runData = createPerfRunData(file, false, typeof(NoBodyDocMaker).AssemblyQualifiedName);
WriteLineDocTask wldt = new WriteLineDocTask(runData);
wldt.DoLogic();
wldt.Dispose();
doReadTest(file, FileType.PLAIN, "title", "date", null);
}
public void TestCharsReplace()
{
// WriteLineDocTask replaced only \t characters w/ a space, since that's its
// separator char. However, it didn't replace newline characters, which
// resulted in errors in LineDocSource.
FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line"));
PerfRunData runData = createPerfRunData(file, false, typeof(NewLinesDocMaker).AssemblyQualifiedName);
WriteLineDocTask wldt = new WriteLineDocTask(runData);
wldt.DoLogic();
wldt.Dispose();
doReadTest(file, FileType.PLAIN, "title text", "date text", "body text two");
}
public void TestCategoryLines()
{
// WriteLineDocTask replaced only \t characters w/ a space, since that's its
// separator char. However, it didn't replace newline characters, which
// resulted in errors in LineDocSource.
FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "two-lines-each.txt"));
PerfRunData runData = createPerfRunData(file, typeof(WriteLineCategoryDocMaker).AssemblyQualifiedName);
WriteLineDocTask wldt = new WriteEnwikiLineDocTask(runData);
for (int i = 0; i < 4; i++)
{ // four times so that each file should have 2 lines.
wldt.DoLogic();
}
wldt.Dispose();
doReadTest(file, "title text", "date text", "body text");
}
public override void Setup()
{
base.Setup();
//check to make sure either the doc is being stored
PerfRunData data = RunData;
if (data.Config.Get("doc.stored", false) == false)
{
throw new Exception("doc.stored must be set to true");
}
if (data.Config.Get("doc.term.vector.offsets", false) == false)
{
throw new Exception("doc.term.vector.offsets must be set to true");
}
if (data.Config.Get("doc.term.vector.positions", false) == false)
{
throw new Exception("doc.term.vector.positions must be set to true");
}
}
public static void Main(string[] args)
{
var p = InitProps();
Config conf = new Config(p);
PerfRunData runData = new PerfRunData(conf);
// 1. top sequence
TaskSequence top = new TaskSequence(runData, null, null, false); // top level, not parallel
// 2. task to create the index
CreateIndexTask create = new CreateIndexTask(runData);
top.AddTask(create);
// 3. task seq to add 500 docs (order matters - top to bottom - add seq to top, only then add to seq)
TaskSequence seq1 = new TaskSequence(runData, "AddDocs", top, false);
seq1.SetRepetitions(500);
seq1.SetNoChildReport();
top.AddTask(seq1);
// 4. task to add the doc
AddDocTask addDoc = new AddDocTask(runData);
//addDoc.setParams("1200"); // doc size limit if supported
seq1.AddTask(addDoc); // order matters 9see comment above)
// 5. task to close the index
CloseIndexTask close = new CloseIndexTask(runData);
top.AddTask(close);
// task to report
RepSumByNameTask rep = new RepSumByNameTask(runData);
top.AddTask(rep);
// print algorithm
Console.WriteLine(top.ToString());
// execute
top.DoLogic();
}
public override int DoLogic()
{
PerfRunData runData = RunData;
Config config = runData.Config;
IndexCommit ic;
if (commitUserData != null)
{
ic = OpenReaderTask.FindIndexCommit(runData.Directory, commitUserData);
}
else
{
ic = null;
}
IndexWriter writer = CreateIndexTask.ConfigureWriter(config, runData, OpenMode.APPEND, ic);
runData.IndexWriter = writer;
return(1);
}
