public override int DoLogic()
{
PerfRunData runData = RunData;
// Get initial reader
IndexWriter w = runData.IndexWriter;
if (w == null)
{
throw new Exception("please open the writer before invoking NearRealtimeReader");
}
if (runData.GetIndexReader() != null)
{
throw new Exception("please close the existing reader before invoking NearRealtimeReader");
}
long t = J2N.Time.CurrentTimeMilliseconds();
DirectoryReader r = DirectoryReader.Open(w, true);
runData.SetIndexReader(r);
// Transfer our reference to runData
r.DecRef();
// TODO: gather basic metrics for reporting -- eg mean,
// stddev, min/max reopen latencies
// Parent sequence sets stopNow
reopenCount = 0;
while (!Stop)
{
long waitForMsec = (pauseMSec - (J2N.Time.CurrentTimeMilliseconds() - t));
if (waitForMsec > 0)
{
Thread.Sleep((int)waitForMsec);
//System.out.println("NRT wait: " + waitForMsec + " msec");
}
t = J2N.Time.CurrentTimeMilliseconds();
DirectoryReader newReader = DirectoryReader.OpenIfChanged(r);
if (newReader != null)
{
int delay = (int)(J2N.Time.CurrentTimeMilliseconds() - t);
if (reopenTimes.Length == reopenCount)
{
reopenTimes = ArrayUtil.Grow(reopenTimes, 1 + reopenCount);
}
reopenTimes[reopenCount++] = delay;
// TODO: somehow we need to enable warming, here
runData.SetIndexReader(newReader);
// Transfer our reference to runData
newReader.DecRef();
r = newReader;
}
}
Stop = false;
return(reopenCount);
}
internal static void SetIndexReaderSearcher()
{
lock (typeof(PatternsForEachTokenLucene))
{
try
{
FSDirectory index = NIOFSDirectory.Open(indexDir);
if (reader == null)
{
reader = DirectoryReader.Open(index);
searcher = new IndexSearcher(reader);
}
else
{
DirectoryReader newreader = DirectoryReader.OpenIfChanged(reader);
if (newreader != null)
{
reader.Close();
reader = newreader;
searcher = new IndexSearcher(reader);
}
}
}
catch (IOException e)
{
throw new Exception(e);
}
}
}
public IEnumerable <(string Term, T Data)> Search(string[] searchTerms, int maxHits = 100)
{
if (!_reader.IsCurrent())
{
using (_reader)
{
_reader = DirectoryReader.OpenIfChanged(_reader);
_searcher = new IndexSearcher(_reader);
}
}
MultiPhraseQuery query = new MultiPhraseQuery();
foreach (string searchTerm in searchTerms)
{
query.Add(new Term(TermsFieldName, searchTerm));
}
TopDocs hits = _searcher.Search(query, maxHits);
foreach (ScoreDoc topDoc in hits.ScoreDocs)
{
Document document = _reader.Document(topDoc.Doc);
string serializedData = document.GetField(DataFieldName).GetStringValue();
T t = _serializer.Deserialize <T>(serializedData);
string term = document.GetField(TermsFieldName).GetStringValue();
yield return(term, t);
}
}
protected override SearcherAndTaxonomy RefreshIfNeeded(SearcherAndTaxonomy @ref)
{
// Must re-open searcher first, otherwise we may get a
// new reader that references ords not yet known to the
// taxonomy reader:
IndexReader r = @ref.searcher.IndexReader;
IndexReader newReader = DirectoryReader.OpenIfChanged((DirectoryReader)r);
if (newReader == null)
{
return(null);
}
else
{
var tr = TaxonomyReader.OpenIfChanged(@ref.taxonomyReader);
if (tr == null)
{
@ref.taxonomyReader.IncRef();
tr = @ref.taxonomyReader;
}
else if (taxoWriter != null && taxoWriter.TaxonomyEpoch != taxoEpoch)
{
IOUtils.Close(newReader, tr);
throw new ThreadStateException("DirectoryTaxonomyWriter.replaceTaxonomy was called, which is not allowed when using SearcherTaxonomyManager");
}
return(new SearcherAndTaxonomy(SearcherManager.GetSearcher(searcherFactory, newReader), tr));
}
}
private void ReopenReader()
{
DirectoryReader newReader = DirectoryReader.OpenIfChanged(reader);
assertNotNull(newReader);
reader.Dispose();
reader = newReader;
}
public override int DoLogic()
{
DirectoryReader r = RunData.GetIndexReader();
DirectoryReader nr = DirectoryReader.OpenIfChanged(r);
if (nr != null)
{
RunData.SetIndexReader(nr);
nr.DecRef();
}
r.DecRef();
return(1);
}
private static DirectoryReader RefreshReader(DirectoryReader reader)
{
DirectoryReader oldReader = reader;
reader = DirectoryReader.OpenIfChanged(reader);
if (reader != null)
{
oldReader.Dispose();
return(reader);
}
else
{
return(oldReader);
}
}
protected override IndexSearcher RefreshIfNeeded(IndexSearcher referenceToRefresh)
{
IndexReader r = referenceToRefresh.IndexReader;
Debug.Assert(r is DirectoryReader, "searcher's IndexReader should be a DirectoryReader, but got " + r);
IndexReader newReader = DirectoryReader.OpenIfChanged((DirectoryReader)r);
if (newReader == null)
{
return(null);
}
else
{
return(GetSearcher(SearcherFactory, newReader));
}
}
public bool? Call()
{
if (indexReader == null)
{
indexReader = DirectoryReader.Open(indexDir);
lastIndexGeneration = indexReader.IndexCommit.Generation;
taxoReader = new DirectoryTaxonomyReader(taxoDir);
}
else
{
// verify search index
DirectoryReader newReader = DirectoryReader.OpenIfChanged(indexReader);
assertNotNull("should not have reached here if no changes were made to the index", newReader);
long newGeneration = newReader.IndexCommit.Generation;
assertTrue("expected newer generation; current=" + lastIndexGeneration + " new=" + newGeneration, newGeneration > lastIndexGeneration);
indexReader.Dispose();
indexReader = newReader;
lastIndexGeneration = newGeneration;
TestUtil.CheckIndex(indexDir);
// verify taxonomy index
DirectoryTaxonomyReader newTaxoReader = TaxonomyReader.OpenIfChanged(taxoReader);
if (newTaxoReader != null)
{
taxoReader.Dispose();
taxoReader = newTaxoReader;
}
TestUtil.CheckIndex(taxoDir);
// verify faceted search
int id = int.Parse(indexReader.IndexCommit.UserData[VERSION_ID], NumberStyles.HexNumber);
IndexSearcher searcher = new IndexSearcher(indexReader);
FacetsCollector fc = new FacetsCollector();
searcher.Search(new MatchAllDocsQuery(), fc);
Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc);
assertEquals(1, (int)facets.GetSpecificValue("A", id.ToString("X")));
DrillDownQuery drillDown = new DrillDownQuery(config);
drillDown.Add("A", id.ToString("X"));
TopDocs docs = searcher.Search(drillDown, 10);
assertEquals(1, docs.TotalHits);
}
return null;
}
/// <summary>
/// Opens the index reader.
/// </summary>
private void OpenReader()
{
if (_reader == null)
{
_reader = DirectoryReader.Open(Directory);
_indexSearcher = new IndexSearcher(_reader);
}
else
{
DirectoryReader newReader =
DirectoryReader.OpenIfChanged(_reader);
if (newReader != null)
{
_reader.Dispose();
_reader = newReader;
_indexSearcher = new IndexSearcher(_reader);
}
}
}
public void Call()
{
if (reader is null)
{
reader = DirectoryReader.Open(indexDir);
lastGeneration = reader.IndexCommit.Generation;
}
else
{
DirectoryReader newReader = DirectoryReader.OpenIfChanged(reader);
assertNotNull("should not have reached here if no changes were made to the index", newReader);
long newGeneration = newReader.IndexCommit.Generation;
assertTrue("expected newer generation; current=" + lastGeneration + " new=" + newGeneration, newGeneration > lastGeneration);
reader.Dispose();
reader = newReader;
lastGeneration = newGeneration;
TestUtil.CheckIndex(indexDir);
}
}
private async Task <IndexReader> GetReaderAsync()
{
try
{
await _semaphoreSlim.WaitAsync();
var newReader = DirectoryReader.OpenIfChanged(_indexReader);
if (newReader != null)
{
_indexReader.Dispose();
_indexReader = newReader;
}
return(_indexReader);
}
finally
{
_semaphoreSlim.Release();
}
}
/// <exception cref="System.IO.IOException"/>
internal virtual void SetIndexReaderSearcher()
{
FSDirectory index = FSDirectory.Open(indexDir);
if (reader == null)
{
reader = DirectoryReader.Open(index);
searcher = new IndexSearcher(reader);
}
else
{
DirectoryReader newreader = DirectoryReader.OpenIfChanged(reader);
if (newreader != null)
{
reader.Close();
reader = newreader;
searcher = new IndexSearcher(reader);
}
}
}
public virtual void TestNRTAndCommit()
{
Directory dir = NewDirectory();
NRTCachingDirectory cachedDir = new NRTCachingDirectory(dir, 2.0, 25.0);
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
RandomIndexWriter w = new RandomIndexWriter(Random, cachedDir, conf);
LineFileDocs docs = new LineFileDocs(Random, DefaultCodecSupportsDocValues);
int numDocs = TestUtil.NextInt32(Random, 100, 400);
if (Verbose)
{
Console.WriteLine("TEST: numDocs=" + numDocs);
}
IList <BytesRef> ids = new List <BytesRef>();
DirectoryReader r = null;
for (int docCount = 0; docCount < numDocs; docCount++)
{
Document doc = docs.NextDoc();
ids.Add(new BytesRef(doc.Get("docid")));
w.AddDocument(doc);
if (Random.Next(20) == 17)
{
if (r == null)
{
r = DirectoryReader.Open(w.IndexWriter, false);
}
else
{
DirectoryReader r2 = DirectoryReader.OpenIfChanged(r);
if (r2 != null)
{
r.Dispose();
r = r2;
}
}
Assert.AreEqual(1 + docCount, r.NumDocs);
IndexSearcher s = NewSearcher(r);
// Just make sure search can run; we can't assert
// totHits since it could be 0
TopDocs hits = s.Search(new TermQuery(new Term("body", "the")), 10);
// System.out.println("tot hits " + hits.totalHits);
}
}
if (r != null)
{
r.Dispose();
}
// Close should force cache to clear since all files are sync'd
w.Dispose();
string[] cachedFiles = cachedDir.ListCachedFiles();
foreach (string file in cachedFiles)
{
Console.WriteLine("FAIL: cached file " + file + " remains after sync");
}
Assert.AreEqual(0, cachedFiles.Length);
r = DirectoryReader.Open(dir);
foreach (BytesRef id in ids)
{
Assert.AreEqual(1, r.DocFreq(new Term("docid", id)));
}
r.Dispose();
cachedDir.Dispose();
docs.Dispose();
}
/// <summary>
/// An <see cref="IndexReader"/> is an instance of the index at a given point in time
/// We need to update this Reader by reopen the <see cref="IndexReader"/>
/// Maybe change this method later ?
/// </summary>
/// <param name="appName"></param>
private void UpdateReader(string appName)
{
_readers[appName] = DirectoryReader.OpenIfChanged(_readers[appName] as DirectoryReader) ?? _readers[appName];
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
/// Returns an instance of the index reader based on the writer supporting NRTS for the main
/// index. Please note that this method will also open a taxonomy reader at the same time, if it
/// is not yet open.
/// </summary>
///
/// <remarks> Semantika d.o.o.,. </remarks>
///
/// <param name="indexType"> Type of the index. </param>
///
/// <returns>
/// Main index reader instance containing the latest NRTS changes and a coresponding taxonomy
/// reader.
/// </returns>
////////////////////////////////////////////////////////////////////////////////////////////////////
public IndexReader GetIndexReader(IndexType indexType)
{
var tmpIndexType = ApplyIndexingMode(indexType);
var currentManagementObjects = m_managementObjects[tmpIndexType];
//Make sure that only one thread at a time is manipulating reader instances
//lock (m_indexIOLock)
{
//Check if we have an instance of the main index reader
if (currentManagementObjects.Reader == null)
{
if (IsReadOnly)
{
currentManagementObjects.Reader = DirectoryReader.Open(currentManagementObjects.Directory);
}
else
{
//If we do not yet have an instance of the reader, we need to make sure we have a functional writer first
if (currentManagementObjects.Writer == null || currentManagementObjects.Writer.IsClosed)
{
//If we did not yet have an open writer, we need to open one.
EnsureOpenWriter();
}
//Since we had no open reader, we can now open a new one which will include all of the NRTS changes
currentManagementObjects.Reader = DirectoryReader.Open(currentManagementObjects.Writer, true);
}
}
else
{
if (!IsReadOnly)
{
//We already have a previous instance of the reader - make sure that includes all of the changes done to the index in the meantime
DirectoryReader newReader;
try
{
newReader = DirectoryReader.OpenIfChanged((DirectoryReader)currentManagementObjects.Reader, currentManagementObjects.Writer, true);
}
catch (IOException)
{
if (currentManagementObjects.Writer == null || currentManagementObjects.Writer.IsClosed)
{
//If we did not yet have an open writer, we need to open one.
EnsureOpenWriter();
}
newReader = DirectoryReader.Open(currentManagementObjects.Writer, true);
}
if (newReader != null)
{
//There was a change in the index since the last time reader was accessed - create a new instance and close the old one
if (currentManagementObjects.CurrentReaderUsers == 0)
{
//No one is using the reader at the moment - close it
currentManagementObjects.Reader.Dispose();
}
else
{
//We have searchers using the reader - add it to cleanup queue
currentManagementObjects.OldReaders[currentManagementObjects.Reader] = currentManagementObjects.CurrentReaderUsers;
}
currentManagementObjects.CurrentReaderUsers = 0;
currentManagementObjects.Reader = newReader;
//Inform listeners that the index was refreshed
IndexRefreshed?.Invoke(this, new EventArgs());
}
}
}
currentManagementObjects.CurrentReaderUsers++;
//Return the created / refreshed instance of the reader to the caller and exit critical section
return(currentManagementObjects.Reader);
}
}
/// <summary>
/// Implements the opening of a new <seealso cref="DirectoryTaxonomyReader"/> instance if
/// the taxonomy has changed.
///
/// <para>
/// <b>NOTE:</b> the returned <seealso cref="DirectoryTaxonomyReader"/> shares the
/// ordinal and category caches with this reader. This is not expected to cause
/// any issues, unless the two instances continue to live. The reader
/// guarantees that the two instances cannot affect each other in terms of
/// correctness of the caches, however if the size of the cache is changed
/// through <seealso cref="#setCacheSize(int)"/>, it will affect both reader instances.
/// </para>
/// </summary>
protected override TaxonomyReader DoOpenIfChanged()
{
EnsureOpen();
// This works for both NRT and non-NRT readers (i.e. an NRT reader remains NRT).
var r2 = DirectoryReader.OpenIfChanged(indexReader);
if (r2 == null)
{
return(null); // no changes, nothing to do
}
// check if the taxonomy was recreated
bool success = false;
try
{
bool recreated = false;
if (taxoWriter == null)
{
// not NRT, check epoch from commit data
string t1 = indexReader.IndexCommit.UserData[DirectoryTaxonomyWriter.INDEX_EPOCH];
string t2 = r2.IndexCommit.UserData[DirectoryTaxonomyWriter.INDEX_EPOCH];
if (t1 == null)
{
if (t2 != null)
{
recreated = true;
}
}
else if (!t1.Equals(t2))
{
// t1 != null and t2 cannot be null b/c DirTaxoWriter always puts the commit data.
// it's ok to use String.equals because we require the two epoch values to be the same.
recreated = true;
}
}
else
{
// NRT, compare current taxoWriter.epoch() vs the one that was given at construction
if (taxoEpoch != taxoWriter.TaxonomyEpoch)
{
recreated = true;
}
}
DirectoryTaxonomyReader newtr;
if (recreated)
{
// if recreated, do not reuse anything from this instace. the information
// will be lazily computed by the new instance when needed.
newtr = new DirectoryTaxonomyReader(r2, taxoWriter, null, null, null);
}
else
{
newtr = new DirectoryTaxonomyReader(r2, taxoWriter, ordinalCache, categoryCache, taxoArrays);
}
success = true;
return(newtr);
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(r2);
}
}
}
