private void doTestIndexProperties(bool setIndexProps, bool indexPropsVal, int numExpectedResults) { Dictionary <string, string> props = new Dictionary <string, string>(); // Indexing configuration. props["analyzer"] = typeof(WhitespaceAnalyzer).AssemblyQualifiedName; props["content.source"] = typeof(OneDocSource).AssemblyQualifiedName; props["directory"] = "RAMDirectory"; if (setIndexProps) { props["doc.index.props"] = indexPropsVal.ToString(); } // Create PerfRunData Config config = new Config(props); PerfRunData runData = new PerfRunData(config); TaskSequence tasks = new TaskSequence(runData, TestName, null, false); tasks.AddTask(new CreateIndexTask(runData)); tasks.AddTask(new AddDocTask(runData)); tasks.AddTask(new CloseIndexTask(runData)); tasks.DoLogic(); IndexReader reader = DirectoryReader.Open(runData.Directory); IndexSearcher searcher = NewSearcher(reader); TopDocs td = searcher.Search(new TermQuery(new Term("key", "value")), 10); assertEquals(numExpectedResults, td.TotalHits); reader.Dispose(); }
public static IndexWriter ConfigureWriter(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit) { IndexWriterConfig iwc = CreateWriterConfig(config, runData, mode, commit); string infoStreamVal = config.Get("writer.info.stream", null); if (infoStreamVal != null) { if (infoStreamVal.Equals("SystemOut", StringComparison.Ordinal)) { iwc.SetInfoStream(Console.Out); } else if (infoStreamVal.Equals("SystemErr", StringComparison.Ordinal)) { iwc.SetInfoStream(Console.Error); } else { FileInfo f = new FileInfo(infoStreamVal); iwc.SetInfoStream(new StreamWriter(new FileStream(f.FullName, FileMode.Create, FileAccess.Write), Encoding.GetEncoding(0))); } } IndexWriter writer = new IndexWriter(runData.Directory, iwc); return(writer); }
public override int DoLogic() { PerfRunData runData = RunData; // Get initial reader IndexWriter w = runData.IndexWriter; if (w == null) { throw new Exception("please open the writer before invoking NearRealtimeReader"); } if (runData.GetIndexReader() != null) { throw new Exception("please close the existing reader before invoking NearRealtimeReader"); } long t = J2N.Time.CurrentTimeMilliseconds(); DirectoryReader r = DirectoryReader.Open(w, true); runData.SetIndexReader(r); // Transfer our reference to runData r.DecRef(); // TODO: gather basic metrics for reporting -- eg mean, // stddev, min/max reopen latencies // Parent sequence sets stopNow reopenCount = 0; while (!Stop) { long waitForMsec = (pauseMSec - (J2N.Time.CurrentTimeMilliseconds() - t)); if (waitForMsec > 0) { Thread.Sleep((int)waitForMsec); //System.out.println("NRT wait: " + waitForMsec + " msec"); } t = J2N.Time.CurrentTimeMilliseconds(); DirectoryReader newReader = DirectoryReader.OpenIfChanged(r); if (newReader != null) { int delay = (int)(J2N.Time.CurrentTimeMilliseconds() - t); if (reopenTimes.Length == reopenCount) { reopenTimes = ArrayUtil.Grow(reopenTimes, 1 + reopenCount); } reopenTimes[reopenCount++] = delay; // TODO: somehow we need to enable warming, here runData.SetIndexReader(newReader); // Transfer our reference to runData newReader.DecRef(); r = newReader; } } Stop = false; return(reopenCount); }
public PerfTask(PerfRunData runData) : this() { this.runData = runData; Config config = runData.Config; this.maxDepthLogStart = config.Get("task.max.depth.log", 0); string logStepAtt = "log.step"; string taskLogStepAtt = "log.step." + name; if (config.Get(taskLogStepAtt, null) != null) { logStepAtt = taskLogStepAtt; } // It's important to read this from Config, to support vals-by-round. m_logStep = config.Get(logStepAtt, DEFAULT_LOG_STEP); // To avoid the check 'if (logStep > 0)' in tearDown(). This effectively // turns logging off. if (m_logStep <= 0) { m_logStep = int.MaxValue; } }
public override int DoLogic() { PerfRunData runData = RunData; runData.TaxonomyWriter = new DirectoryTaxonomyWriter(runData.TaxonomyDir); return(1); }
private void doLogStepTest(bool setLogStep, int logStepVal, bool setTaskLogStep, int taskLogStepVal, int expLogStepValue) { PerfRunData runData = createPerfRunData(setLogStep, logStepVal, setTaskLogStep, taskLogStepVal); MyPerfTask mpt = new MyPerfTask(runData); assertEquals(expLogStepValue, mpt.getLogStep()); }
public WriteEnwikiLineDocTask(PerfRunData runData) : base(runData) { Stream @out = StreamUtils.GetOutputStream(CategoriesLineFile(new FileInfo(m_fname))); categoryLineFileOut = new StreamWriter(@out, Encoding.UTF8); WriteHeader(categoryLineFileOut); }
public void TestNoParams() { PerfRunData runData = createPerfRunData(); new CreateIndexTask(runData).DoLogic(); new CommitIndexTask(runData).DoLogic(); new CloseIndexTask(runData).DoLogic(); }
public void TestNoMergeScheduler() { PerfRunData runData = createPerfRunData(null); runData.Config.Set("merge.scheduler", typeof(NoMergeScheduler).AssemblyQualifiedName); new CreateIndexTask(runData).DoLogic(); new CloseIndexTask(runData).DoLogic(); }
public void TestNoDeletionPolicy() { PerfRunData runData = createPerfRunData(null); runData.Config.Set("deletion.policy", typeof(NoDeletionPolicy).AssemblyQualifiedName); new CreateIndexTask(runData).DoLogic(); new CloseIndexTask(runData).DoLogic(); }
public override int DoLogic() { PerfRunData runData = RunData; Config config = runData.Config; runData.IndexWriter = ConfigureWriter(config, runData, OpenMode.CREATE, null); return(1); }
public void TestInfoStream_File() { FileInfo outFile = new FileInfo(Path.Combine(getWorkDir().FullName, "infoStreamTest")); PerfRunData runData = createPerfRunData(outFile.FullName); new CreateIndexTask(runData).DoLogic(); new CloseIndexTask(runData).DoLogic(); assertTrue(new FileInfo(outFile.FullName).Length > 0); }
public override void Setup() { base.Setup(); //check to make sure either the doc is being stored PerfRunData runData = RunData; Config config = runData.Config; m_clnName = config.Get("collector.class", ""); }
public override int DoLogic() { PerfRunData runData = RunData; DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(runData.TaxonomyDir); runData.SetTaxonomyReader(taxoReader); // We transfer reference to the run data taxoReader.DecRef(); return(1); }
private readonly object lineFileLock = new object(); // LUCENENET specific - lock to ensure writes don't collide for this instance public WriteLineDocTask(PerfRunData runData) : base(runData) { Config config = runData.Config; m_fname = config.Get("line.file.out", null); if (m_fname == null) { throw new ArgumentException("line.file.out must be set"); } Stream @out = StreamUtils.GetOutputStream(new FileInfo(m_fname)); lineFileOut = new StreamWriter(@out, Encoding.UTF8); docMaker = runData.DocMaker; // init fields string f2r = config.Get("line.fields", null); if (f2r == null) { fieldsToWrite = DEFAULT_FIELDS; } else { if (f2r.IndexOf(SEP) >= 0) { throw new ArgumentException("line.fields " + f2r + " should not contain the separator char: " + SEP); } fieldsToWrite = f2r.Split(',').TrimEnd(); } // init sufficient fields sufficientFields = new bool[fieldsToWrite.Length]; string suff = config.Get("sufficient.fields", DEFAULT_SUFFICIENT_FIELDS); if (",".Equals(suff, StringComparison.Ordinal)) { checkSufficientFields = false; } else { checkSufficientFields = true; ISet <string> sf = new JCG.HashSet <string>(suff.Split(',').TrimEnd()); for (int i = 0; i < fieldsToWrite.Length; i++) { if (sf.Contains(fieldsToWrite[i])) { sufficientFields[i] = true; } } } WriteHeader(lineFileOut); }
protected ReadTask(PerfRunData runData) // LUCENENET: CA1012: Abstract types should not have constructors (marked protected) : base(runData) { if (WithSearch) { queryMaker = GetQueryMaker(); } else { queryMaker = null; } }
public ReadTask(PerfRunData runData) : base(runData) { if (WithSearch) { queryMaker = GetQueryMaker(); } else { queryMaker = null; } }
public void TestEmptyTitle() { FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line")); PerfRunData runData = createPerfRunData(file, false, typeof(NoTitleDocMaker).AssemblyQualifiedName); WriteLineDocTask wldt = new WriteLineDocTask(runData); wldt.DoLogic(); wldt.Dispose(); doReadTest(file, FileType.PLAIN, "", "date", "body"); }
public override void Setup() { base.Setup(); //check to make sure either the doc is being stored PerfRunData data = RunData; if (data.Config.Get("doc.stored", false) == false) { throw new Exception("doc.stored must be set to true"); } m_maxDocCharsToAnalyze = data.Config.Get("highlighter.maxDocCharsToAnalyze", Highlighter.DEFAULT_MAX_CHARS_TO_ANALYZE); }
public TaskSequence(PerfRunData runData, String name, TaskSequence parent, bool parallel) : base(runData) { collapsable = (name == null); name = (name != null ? name : (parallel ? "Par" : "Seq")); SetName(name); SetSequenceName(); this.parent = parent; this.parallel = parallel; tasks = new List <PerfTask>(); logByTimeMsec = runData.Config.Get("report.time.step.msec", 0); }
public void TestRegularFile() { // Create a document in regular format. FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line")); PerfRunData runData = createPerfRunData(file, false, typeof(WriteLineDocMaker).AssemblyQualifiedName); WriteLineDocTask wldt = new WriteLineDocTask(runData); wldt.DoLogic(); wldt.Dispose(); doReadTest(file, FileType.PLAIN, "title", "date", "body"); }
public void TestMultiThreaded() { FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line")); PerfRunData runData = createPerfRunData(file, false, typeof(ThreadingDocMaker).AssemblyQualifiedName); ThreadJob[] threads = new ThreadJob[10]; using (WriteLineDocTask wldt = new WriteLineDocTask(runData)) { for (int i = 0; i < threads.Length; i++) { threads[i] = new ThreadAnonymousHelper("t" + i, wldt); } foreach (ThreadJob t in threads) { t.Start(); } foreach (ThreadJob t in threads) { t.Join(); } } // wldt.Dispose(); ISet <String> ids = new JCG.HashSet <string>(); TextReader br = new StreamReader(new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.None), Encoding.UTF8); try { String line = br.ReadLine(); assertHeaderLine(line); // header line is written once, no matter how many threads there are for (int i = 0; i < threads.Length; i++) { line = br.ReadLine(); assertNotNull($"line for index {i} is missing", line); // LUCENENET specific - ensure the line is there before splitting String[] parts = line.Split(WriteLineDocTask.SEP).TrimEnd(); assertEquals(line, 3, parts.Length); // check that all thread names written are the same in the same line String tname = parts[0].Substring(parts[0].IndexOf('_')); ids.add(tname); assertEquals(tname, parts[1].Substring(parts[1].IndexOf('_'))); assertEquals(tname, parts[2].Substring(parts[2].IndexOf('_'))); } // only threads.length lines should exist assertNull(br.ReadLine()); assertEquals(threads.Length, ids.size()); } finally { br.Dispose(); } }
public void TestCommitData() { PerfRunData runData = createPerfRunData(); new CreateIndexTask(runData).DoLogic(); CommitIndexTask task = new CommitIndexTask(runData); task.SetParams("params"); task.DoLogic(); SegmentInfos infos = new SegmentInfos(); infos.Read(runData.Directory); assertEquals("params", infos.UserData[OpenReaderTask.USER_DATA]); new CloseIndexTask(runData).DoLogic(); }
private void assertIndex(PerfRunData runData) { Store.Directory taskDir = runData.Directory; assertSame(typeof(RAMDirectory), taskDir.GetType()); IndexReader r = DirectoryReader.Open(taskDir); try { assertEquals(10, r.NumDocs); } finally { r.Dispose(); } }
public void TestEmptyBody() { // WriteLineDocTask threw away documents w/ no BODY element, even if they // had a TITLE element (LUCENE-1755). It should throw away documents if they // don't have BODY nor TITLE FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line")); PerfRunData runData = createPerfRunData(file, false, typeof(NoBodyDocMaker).AssemblyQualifiedName); WriteLineDocTask wldt = new WriteLineDocTask(runData); wldt.DoLogic(); wldt.Dispose(); doReadTest(file, FileType.PLAIN, "title", "date", null); }
public void TestCharsReplace() { // WriteLineDocTask replaced only \t characters w/ a space, since that's its // separator char. However, it didn't replace newline characters, which // resulted in errors in LineDocSource. FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "one-line")); PerfRunData runData = createPerfRunData(file, false, typeof(NewLinesDocMaker).AssemblyQualifiedName); WriteLineDocTask wldt = new WriteLineDocTask(runData); wldt.DoLogic(); wldt.Dispose(); doReadTest(file, FileType.PLAIN, "title text", "date text", "body text two"); }
public void TestCategoryLines() { // WriteLineDocTask replaced only \t characters w/ a space, since that's its // separator char. However, it didn't replace newline characters, which // resulted in errors in LineDocSource. FileInfo file = new FileInfo(Path.Combine(getWorkDir().FullName, "two-lines-each.txt")); PerfRunData runData = createPerfRunData(file, typeof(WriteLineCategoryDocMaker).AssemblyQualifiedName); WriteLineDocTask wldt = new WriteEnwikiLineDocTask(runData); for (int i = 0; i < 4; i++) { // four times so that each file should have 2 lines. wldt.DoLogic(); } wldt.Dispose(); doReadTest(file, "title text", "date text", "body text"); }
public override void Setup() { base.Setup(); //check to make sure either the doc is being stored PerfRunData data = RunData; if (data.Config.Get("doc.stored", false) == false) { throw new Exception("doc.stored must be set to true"); } if (data.Config.Get("doc.term.vector.offsets", false) == false) { throw new Exception("doc.term.vector.offsets must be set to true"); } if (data.Config.Get("doc.term.vector.positions", false) == false) { throw new Exception("doc.term.vector.positions must be set to true"); } }
public static void Main(string[] args) { var p = InitProps(); Config conf = new Config(p); PerfRunData runData = new PerfRunData(conf); // 1. top sequence TaskSequence top = new TaskSequence(runData, null, null, false); // top level, not parallel // 2. task to create the index CreateIndexTask create = new CreateIndexTask(runData); top.AddTask(create); // 3. task seq to add 500 docs (order matters - top to bottom - add seq to top, only then add to seq) TaskSequence seq1 = new TaskSequence(runData, "AddDocs", top, false); seq1.SetRepetitions(500); seq1.SetNoChildReport(); top.AddTask(seq1); // 4. task to add the doc AddDocTask addDoc = new AddDocTask(runData); //addDoc.setParams("1200"); // doc size limit if supported seq1.AddTask(addDoc); // order matters 9see comment above) // 5. task to close the index CloseIndexTask close = new CloseIndexTask(runData); top.AddTask(close); // task to report RepSumByNameTask rep = new RepSumByNameTask(runData); top.AddTask(rep); // print algorithm Console.WriteLine(top.ToString()); // execute top.DoLogic(); }
public override int DoLogic() { PerfRunData runData = RunData; Config config = runData.Config; IndexCommit ic; if (commitUserData != null) { ic = OpenReaderTask.FindIndexCommit(runData.Directory, commitUserData); } else { ic = null; } IndexWriter writer = CreateIndexTask.ConfigureWriter(config, runData, OpenMode.APPEND, ic); runData.IndexWriter = writer; return(1); }