public DocFieldProcessor(DocumentsWriter docWriter, DocFieldConsumer consumer)
{
this.docWriter = docWriter;
this.consumer = consumer;
consumer.SetFieldInfos(fieldInfos);
fieldsWriter = new StoredFieldsWriter(docWriter, fieldInfos);
}
internal override void closeDocStore(DocumentsWriter.FlushState state)
{
lock (this)
{
int inc = state.numDocsInStore - lastDocID;
if (inc > 0)
{
initFieldsWriter();
fill(state.numDocsInStore - docWriter.GetDocStoreOffset());
}
if (fieldsWriter != null)
{
fieldsWriter.Close();
fieldsWriter = null;
lastDocID = 0;
System.Diagnostics.Debug.Assert(state.docStoreSegmentName != null);
string fdtFile = state.docStoreSegmentName + "." + IndexFileNames.FIELDS_EXTENSION;
string fdxFile = state.docStoreSegmentName + "." + IndexFileNames.FIELDS_INDEX_EXTENSION;
state.flushedFiles[fdtFile] = fdtFile;
state.flushedFiles[fdxFile] = fdxFile;
state.docWriter.RemoveOpenFile(fdtFile);
state.docWriter.RemoveOpenFile(fdxFile);
if (4 + state.numDocsInStore * 8 != state.directory.FileLength(fdxFile))
throw new System.SystemException("after flush: fdx size mismatch: " + state.numDocsInStore + " docs vs " + state.directory.FileLength(fdxFile) + " length in bytes of " + fdxFile);
}
}
}
public override ThreadState GetAndLock(Thread requestingThread, DocumentsWriter documentsWriter)
{
ThreadState threadState = null;
if (ActiveThreadState == 0)
{
lock (this)
{
if (ActiveThreadState == 0)
{
threadState = States[0] = NewThreadState();
return threadState;
}
}
}
Debug.Assert(ActiveThreadState > 0);
for (int i = 0; i < MaxRetry; i++)
{
int ord = Random.Next(ActiveThreadState);
lock (this)
{
threadState = States[ord];
Debug.Assert(threadState != null);
}
if (threadState.TryLock())
{
return threadState;
}
if (Random.Next(20) == 0)
{
break;
}
}
/*
* only try to create a new threadstate if we can not lock the randomly
* selected state. this is important since some tests rely on a single
* threadstate in the single threaded case. Eventually it would be nice if
* we would not have this limitation but for now we just make sure we only
* allocate one threadstate if indexing is single threaded
*/
lock (this)
{
ThreadState newThreadState = NewThreadState();
if (newThreadState != null) // did we get a new state?
{
threadState = States[ActiveThreadState - 1] = newThreadState;
//Debug.Assert(threadState.HeldByCurrentThread);
return threadState;
}
// if no new state is available lock the random one
}
Debug.Assert(threadState != null);
threadState.@Lock();
return threadState;
}
internal int numThreads = 1; // Number of threads that share this instance
#endregion Fields
#region Constructors
public DocumentsWriterThreadState(DocumentsWriter docWriter)
{
this.docWriter = docWriter;
docState = new DocumentsWriter.DocState();
docState.maxFieldLength = docWriter.maxFieldLength;
docState.infoStream = docWriter.infoStream;
docState.similarity = docWriter.similarity;
docState.docWriter = docWriter;
consumer = docWriter.consumer.AddThread(this);
}
public DocumentsWriterThreadState(DocumentsWriter docWriter)
{
this.docWriter = docWriter;
docState = new DocumentsWriter.DocState();
docState.maxFieldLength = docWriter.maxFieldLength;
docState.infoStream = docWriter.infoStream;
docState.similarity = docWriter.similarity;
docState.docWriter = docWriter;
docState.allowMinus1Position = docWriter.writer.GetAllowMinus1Position();
consumer = docWriter.consumer.AddThread(this);
}
internal DocumentsWriterFlushControl(DocumentsWriter documentsWriter, LiveIndexWriterConfig config, BufferedUpdatesStream bufferedUpdatesStream)
{
this.InfoStream_Renamed = config.InfoStream;
this.StallControl = new DocumentsWriterStallControl();
this.PerThreadPool = documentsWriter.PerThreadPool;
this.FlushPolicy = documentsWriter.FlushPolicy;
this.Config = config;
this.HardMaxBytesPerDWPT = config.RAMPerThreadHardLimitMB * 1024 * 1024;
this.DocumentsWriter = documentsWriter;
this.BufferedUpdatesStream = bufferedUpdatesStream;
}
internal DocumentsWriterFlushControl(DocumentsWriter documentsWriter, LiveIndexWriterConfig config, BufferedUpdatesStream bufferedUpdatesStream)
{
this.infoStream = config.InfoStream;
this.stallControl = new DocumentsWriterStallControl();
this.perThreadPool = documentsWriter.perThreadPool;
this.flushPolicy = documentsWriter.flushPolicy;
this.config = config;
this.hardMaxBytesPerDWPT = config.RAMPerThreadHardLimitMB * 1024 * 1024;
this.documentsWriter = documentsWriter;
this.bufferedUpdatesStream = bufferedUpdatesStream;
}
public SegmentWriteState(DocumentsWriter docWriter, Directory directory, System.String segmentName, System.String docStoreSegmentName, int numDocs, int numDocsInStore, int termIndexInterval)
{
this.docWriter = docWriter;
this.directory = directory;
this.segmentName = segmentName;
this.docStoreSegmentName = docStoreSegmentName;
this.numDocs = numDocs;
this.numDocsInStore = numDocsInStore;
this.termIndexInterval = termIndexInterval;
flushedFiles = new Support.Set <string>();
}
internal override void closeDocStore(DocumentsWriter.FlushState state)
{
try
{
one.closeDocStore(state);
}
finally
{
two.closeDocStore(state);
}
}
internal int numThreads = 1; // Number of threads that share this instance
#endregion Fields
#region Constructors
public DocumentsWriterThreadState(DocumentsWriter docWriter)
{
this.docWriter = docWriter;
docState = new DocumentsWriter.DocState();
docState.maxFieldLength = docWriter.maxFieldLength;
docState.infoStream = docWriter.infoStream;
docState.similarity = docWriter.similarity;
docState.docWriter = docWriter;
docState.allowMinus1Position = docWriter.writer.GetAllowMinus1Position();
consumer = docWriter.consumer.AddThread(this);
}
public SegmentWriteState(DocumentsWriter docWriter, Directory directory, System.String segmentName, System.String docStoreSegmentName, int numDocs, int numDocsInStore, int termIndexInterval)
{
this.docWriter = docWriter;
this.directory = directory;
this.segmentName = segmentName;
this.docStoreSegmentName = docStoreSegmentName;
this.numDocs = numDocs;
this.numDocsInStore = numDocsInStore;
this.termIndexInterval = termIndexInterval;
flushedFiles = new Support.Set<string>();
}
public virtual void TestStallControl()
{
int[] numThreads = new int[] { 4 + Random.Next(8), 1 };
int numDocumentsToIndex = 50 + Random.Next(50);
for (int i = 0; i < numThreads.Length; i++)
{
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
MockDirectoryWrapper dir = NewMockDirectory();
// mock a very slow harddisk sometimes here so that flushing is very slow
dir.Throttling = Throttling.SOMETIMES;
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy();
iwc.SetFlushPolicy(flushPolicy);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numThreads[i] == 1 ? 1 : 2);
iwc.SetIndexerThreadPool(threadPool);
// with such a small ram buffer we should be stalled quiet quickly
iwc.SetRAMBufferSizeMB(0.25);
IndexWriter writer = new IndexWriter(dir, iwc);
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads[i], writer, LineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due");
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
if (numThreads[i] == 1)
{
assertFalse("single thread must not block numThreads: " + numThreads[i], docsWriter.flushControl.stallControl.HasBlocked);
}
if (docsWriter.flushControl.peakNetBytes > (2d * iwc.RAMBufferSizeMB * 1024d * 1024d))
{
Assert.IsTrue(docsWriter.flushControl.stallControl.WasStalled);
}
AssertActiveBytesAfter(flushControl);
writer.Dispose(true);
dir.Dispose();
}
}
public virtual void TestFlushDocCount()
{
// LUCENENET specific - disable the test if asserts are not enabled
AssumeTrue("This test requires asserts to be enabled.", Debugging.AssertsEnabled);
int[] numThreads = new int[] { 2 + AtLeast(1), 1 };
for (int i = 0; i < numThreads.Length; i++)
{
int numDocumentsToIndex = 50 + AtLeast(30);
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
Directory dir = NewDirectory();
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetMaxBufferedDocs(2 + AtLeast(10));
iwc.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
Assert.IsTrue(flushPolicy.FlushOnDocCount);
Assert.IsFalse(flushPolicy.FlushOnDeleteTerms);
Assert.IsFalse(flushPolicy.FlushOnRAM);
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(numDocs, writer, lineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads[i]);
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
Assert.IsTrue(flushPolicy.peakDocCountWithoutFlush <= iwc.MaxBufferedDocs, "peak bytes without flush exceeded watermark");
AssertActiveBytesAfter(flushControl);
writer.Dispose();
Assert.AreEqual(0, flushControl.ActiveBytes);
dir.Dispose();
}
}
//internal override void flush(/*IDictionary*/ IDictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> threadsAndFields, DocumentsWriter.FlushState state)
internal override void flush(/*IDictionary*/ IDictionary<object, ICollection<object>> threadsAndFields, DocumentsWriter.FlushState state)
{
//IDictionary oneThreadsAndFields = new Dictionary();
//IDictionary twoThreadsAndFields = new Dictionary();
//IDictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> oneThreadsAndFields = new Dictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>>();
//IDictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> twoThreadsAndFields = new Dictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>>();
IDictionary<object, ICollection<object>> oneThreadsAndFields = new Dictionary<object, ICollection<object>>();
IDictionary<object, ICollection<object>> twoThreadsAndFields = new Dictionary<object, ICollection<object>>();
//IEnumerator it = threadsAndFields.GetEnumerator();
//IEnumerator<KeyValuePair<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>>> it = threadsAndFields.GetEnumerator();
IEnumerator<KeyValuePair<object, ICollection<object>>> it = threadsAndFields.GetEnumerator();
while (it.MoveNext())
{
//KeyValuePair entry = it.Current;
//KeyValuePair<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> entry = it.Current;
KeyValuePair<object, ICollection<object>> entry = it.Current;
//DocFieldConsumersPerThread perThread = (DocFieldConsumersPerThread)entry.Key;
DocFieldConsumersPerThread perThread = (DocFieldConsumersPerThread)entry.Key;
//ICollection fields = (ICollection)entry.Value;
//ICollection<DocFieldConsumerPerField> fields = entry.Value;
ICollection<object> fields = entry.Value;
//IEnumerator fieldsIt = fields.GetEnumerator();
//IEnumerator<DocFieldConsumerPerField> fieldsIt = fields.GetEnumerator();
IEnumerator<object> fieldsIt = fields.GetEnumerator();
//IDictionary oneFields = new Dictionary();
//IDictionary twoFields = new Dictionary();
//IDictionary<DocFieldConsumerPerField, DocFieldConsumerPerField> oneFields = new Dictionary<DocFieldConsumerPerField, DocFieldConsumerPerField>();
//IDictionary<DocFieldConsumerPerField, DocFieldConsumerPerField> twoFields = new Dictionary<DocFieldConsumerPerField, DocFieldConsumerPerField>();
IDictionary<object, object> oneFields = new Dictionary<object, object>();
IDictionary<object, object> twoFields = new Dictionary<object, object>();
while (fieldsIt.MoveNext())
{
DocFieldConsumersPerField perField = (DocFieldConsumersPerField)fieldsIt.Current;
oneFields[perField.one] = perField.one;
twoFields[perField.two] = perField.two;
}
oneThreadsAndFields[perThread.one] = oneFields.Keys;
twoThreadsAndFields[perThread.two] = twoFields.Keys;
}
one.flush(oneThreadsAndFields, state);
two.flush(twoThreadsAndFields, state);
}
public TermsHash(DocumentsWriter docWriter, bool trackAllocations, TermsHashConsumer consumer, TermsHash nextTermsHash)
{
this.docWriter = docWriter;
this.consumer = consumer;
this.nextTermsHash = nextTermsHash;
this.trackAllocations = trackAllocations;
// Why + 4*POINTER_NUM_BYTE below?
// +1: Posting is referenced by postingsFreeList array
// +3: Posting is referenced by hash, which
// targets 25-50% fill factor; approximate this
// as 3X # pointers
bytesPerPosting = consumer.BytesPerPosting() + 4 * DocumentsWriter.POINTER_NUM_BYTE;
postingsFreeChunk = (int)(DocumentsWriter.BYTE_BLOCK_SIZE / bytesPerPosting);
}
public TermsHash(DocumentsWriter docWriter, bool trackAllocations, TermsHashConsumer consumer, TermsHash nextTermsHash)
{
this.docWriter = docWriter;
this.consumer = consumer;
this.nextTermsHash = nextTermsHash;
this.trackAllocations = trackAllocations;
// Why + 4*POINTER_NUM_BYTE below?
// +1: Posting is referenced by postingsFreeList array
// +3: Posting is referenced by hash, which
// targets 25-50% fill factor; approximate this
// as 3X # pointers
bytesPerPosting = consumer.BytesPerPosting() + 4 * DocumentsWriter.POINTER_NUM_BYTE;
postingsFreeChunk = (int) (DocumentsWriter.BYTE_BLOCK_SIZE / bytesPerPosting);
}
internal override void Flush(ICollection<object> threads, DocumentsWriter.FlushState state)
{
IDictionary<object, ICollection<object>> childThreadsAndFields = new Dictionary<object, ICollection<object>>();
IEnumerator<object> it = threads.GetEnumerator();
while (it.MoveNext())
{
DocFieldProcessorPerThread perThread = (DocFieldProcessorPerThread)it.Current;
childThreadsAndFields[perThread.consumer] = perThread.Fields();
perThread.trimFields(state);
}
consumer.flush(childThreadsAndFields, state);
// Important to save after asking consumer to flush so
// consumer can alter the FieldInfo* if necessary. EG,
// FreqProxTermsWriter does this with
// FieldInfo.storePayload.
fieldInfos.Write(state.directory, state.segmentName + ".fnm");
}
public override ThreadState GetAndLock(Thread requestingThread, DocumentsWriter documentsWriter)
{
if (threadBindings.TryGetValue(requestingThread, out ThreadState threadState) && threadState.TryLock())
{
return(threadState);
}
ThreadState minThreadState = null;
/* TODO -- another thread could lock the minThreadState we just got while
* we should somehow prevent this. */
// Find the state that has minimum number of threads waiting
minThreadState = MinContendedThreadState();
if (minThreadState == null || minThreadState.HasQueuedThreads)
{
ThreadState newState = NewThreadState(); // state is already locked if non-null
if (newState != null)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(newState.IsHeldByCurrentThread);
}
threadBindings[requestingThread] = newState;
return(newState);
}
else if (minThreadState == null)
{
/*
* no new threadState available we just take the minContented one
* this must return a valid thread state since we accessed the
* synced context in newThreadState() above.
*/
minThreadState = MinContendedThreadState();
}
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(minThreadState != null, () => "ThreadState is null");
}
minThreadState.@Lock();
return(minThreadState);
}
//internal override void flush(IDictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> threadsAndFields, DocumentsWriter.FlushState state)
internal override void flush(IDictionary<object, ICollection<object>> threadsAndFields, DocumentsWriter.FlushState state)
{
//IDictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> childThreadsAndFields = new Dictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>>();
//IDictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> endChildThreadsAndFields = new Dictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>>();
IDictionary<object, ICollection<object>> childThreadsAndFields = new Dictionary<object, ICollection<object>>();
IDictionary<object, ICollection<object>> endChildThreadsAndFields = new Dictionary<object, ICollection<object>>();
//IEnumerator<KeyValuePair<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>>> it = threadsAndFields.GetEnumerator();
IEnumerator<KeyValuePair<object, ICollection<object>>> it = threadsAndFields.GetEnumerator();
while (it.MoveNext())
{
//KeyValuePair<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> entry = it.Current;
KeyValuePair<object, ICollection<object>> entry = it.Current;
DocInverterPerThread perThread = (DocInverterPerThread)entry.Key;
//ICollection<DocFieldConsumerPerField> fields = entry.Value;
ICollection<object> fields = entry.Value;
//IEnumerator<DocFieldConsumerPerField> fieldsIt = fields.GetEnumerator();
IEnumerator<object> fieldsIt = fields.GetEnumerator();
//IDictionary<DocFieldConsumerPerField, DocFieldConsumerPerField> childFields = new Dictionary<DocFieldConsumerPerField, DocFieldConsumerPerField>();
//IDictionary<DocFieldConsumerPerField, DocFieldConsumerPerField> endChildFields = new Dictionary<DocFieldConsumerPerField, DocFieldConsumerPerField>();
IDictionary<object, object> childFields = new Dictionary<object, object>();
IDictionary<object, object> endChildFields = new Dictionary<object, object>();
while (fieldsIt.MoveNext())
{
DocInverterPerField perField = (DocInverterPerField)fieldsIt.Current;
childFields[perField.consumer] = perField.consumer;
endChildFields[perField.endConsumer] = perField.endConsumer;
}
childThreadsAndFields[perThread.consumer] = childFields.Keys;
// create new collection to provide for deletions in NormsWriter
endChildThreadsAndFields[perThread.endConsumer] = new List<object>(endChildFields.Keys);
}
consumer.flush(childThreadsAndFields, state);
endConsumer.flush(endChildThreadsAndFields, state);
}
public override ThreadState GetAndLock(Thread requestingThread, DocumentsWriter documentsWriter)
{
ThreadState threadState;
ThreadBindings.TryGetValue(requestingThread, out threadState);
if (threadState != null && threadState.TryLock())
{
return threadState;
}
ThreadState minThreadState = null;
/* TODO -- another thread could lock the minThreadState we just got while
we should somehow prevent this. */
// Find the state that has minimum number of threads waiting
minThreadState = MinContendedThreadState();
if (minThreadState == null || minThreadState.HasQueuedThreads)
{
ThreadState newState = NewThreadState(); // state is already locked if non-null
if (newState != null)
{
//Debug.Assert(newState.HeldByCurrentThread);
ThreadBindings[requestingThread] = newState;
return newState;
}
else if (minThreadState == null)
{
/*
* no new threadState available we just take the minContented one
* this must return a valid thread state since we accessed the
* synced context in newThreadState() above.
*/
minThreadState = MinContendedThreadState();
}
}
Debug.Assert(minThreadState != null, "ThreadState is null");
minThreadState.@Lock();
return minThreadState;
}
public CharBlockPool(DocumentsWriter docWriter)
{
InitBlock();
this.docWriter = docWriter;
}
// you cannot subclass this without being in o.a.l.index package anyway, so // the class is already pkg-private... fix me: see LUCENE-4013 public abstract ThreadState GetAndLock(Thread requestingThread, DocumentsWriter documentsWriter);
/// <summary> Initialize the deleter: find all previous commits in
/// the Directory, incref the files they reference, call
/// the policy to let it delete commits. This will remove
/// any files not referenced by any of the commits.
/// </summary>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.StreamWriter infoStream, DocumentsWriter docWriter, HashSet <string> synced)
{
this.docWriter = docWriter;
this.infoStream = infoStream;
this.synced = synced;
if (infoStream != null)
{
Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy);
}
this.policy = policy;
this.directory = directory;
// First pass: walk the files and initialize our ref
// counts:
long currentGen = segmentInfos.Generation;
IndexFileNameFilter filter = IndexFileNameFilter.Filter;
System.String[] files = directory.ListAll();
CommitPoint currentCommitPoint = null;
for (int i = 0; i < files.Length; i++)
{
System.String fileName = files[i];
if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
{
// Add this file to refCounts with initial count 0:
GetRefCount(fileName);
if (fileName.StartsWith(IndexFileNames.SEGMENTS))
{
// This is a commit (segments or segments_N), and
// it's valid (<= the max gen). Load it, then
// incref all files it refers to:
if (infoStream != null)
{
Message("init: load commit \"" + fileName + "\"");
}
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, fileName);
}
catch (System.IO.FileNotFoundException)
{
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
if (infoStream != null)
{
Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
}
sis = null;
}
catch (System.IO.IOException)
{
if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen)
{
throw;
}
else
{
// Most likely we are opening an index that
// has an aborted "future" commit, so suppress
// exc in this case
sis = null;
}
}
if (sis != null)
{
CommitPoint commitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
if (sis.Generation == segmentInfos.Generation)
{
currentCommitPoint = commitPoint;
}
commits.Add(commitPoint);
IncRef(sis, true);
if (lastSegmentInfos == null || sis.Generation > lastSegmentInfos.Generation)
{
lastSegmentInfos = sis;
}
}
}
}
}
if (currentCommitPoint == null)
{
// We did not in fact see the segments_N file
// corresponding to the segmentInfos that was passed
// in. Yet, it must exist, because our caller holds
// the write lock. This can happen when the directory
// listing was stale (eg when index accessed via NFS
// client with stale directory listing cache). So we
// try now to explicitly open this commit point:
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, segmentInfos.GetCurrentSegmentFileName());
}
catch (System.IO.IOException)
{
throw new CorruptIndexException("failed to locate current segments_N file");
}
if (infoStream != null)
{
Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName());
}
currentCommitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
commits.Add(currentCommitPoint);
IncRef(sis, true);
}
// We keep commits list in sorted order (oldest to newest):
commits.Sort();
// Now delete anything with ref count at 0. These are
// presumably abandoned files eg due to crash of
// IndexWriter.
foreach (KeyValuePair <string, RefCount> entry in refCounts)
{
string fileName = entry.Key;
RefCount rc = refCounts[fileName];
if (0 == rc.count)
{
if (infoStream != null)
{
Message("init: removing unreferenced file \"" + fileName + "\"");
}
DeleteFile(fileName);
}
}
// Finally, give policy a chance to remove things on
// startup:
policy.OnInit(commits);
// Always protect the incoming segmentInfos since
// sometime it may not be the most recent commit
Checkpoint(segmentInfos, false);
startingCommitDeleted = currentCommitPoint.IsDeleted;
DeleteCommits();
}
public StoredFieldsWriterPerThread(DocumentsWriter.DocState docState, StoredFieldsWriter storedFieldsWriter)
{
this.storedFieldsWriter = storedFieldsWriter;
this.docState = docState;
localFieldsWriter = new FieldsWriter((IndexOutput) null, (IndexOutput) null, storedFieldsWriter.fieldInfos);
}
public override ThreadState GetAndLock(Thread requestingThread, DocumentsWriter documentsWriter)
{
ThreadState threadState = null;
if (NumThreadStatesActive == 0)
{
lock (this)
{
if (NumThreadStatesActive == 0)
{
threadState = states[0] = NewThreadState();
return(threadState);
}
}
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(NumThreadStatesActive > 0);
}
for (int i = 0; i < maxRetry; i++)
{
int ord = random.Next(NumThreadStatesActive);
lock (this)
{
threadState = states[ord];
if (Debugging.AssertsEnabled)
{
Debugging.Assert(threadState != null);
}
}
if (threadState.TryLock())
{
return(threadState);
}
if (random.Next(20) == 0)
{
break;
}
}
/*
* only try to create a new threadstate if we can not lock the randomly
* selected state. this is important since some tests rely on a single
* threadstate in the single threaded case. Eventually it would be nice if
* we would not have this limitation but for now we just make sure we only
* allocate one threadstate if indexing is single threaded
*/
lock (this)
{
ThreadState newThreadState = NewThreadState();
if (newThreadState != null) // did we get a new state?
{
threadState = states[NumThreadStatesActive - 1] = newThreadState;
//if (Debugging.AssertsEnabled) Debugging.Assert(threadState.HeldByCurrentThread);
return(threadState);
}
// if no new state is available lock the random one
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(threadState != null);
}
threadState.@Lock();
return(threadState);
}
private void InitBlock(DocumentsWriter enclosingInstance)
{
this.enclosingInstance = enclosingInstance;
}
internal override DocConsumer GetChain(DocumentsWriter documentsWriter)
{
/*
This is the current indexing chain:
DocConsumer / DocConsumerPerThread
--> code: DocFieldProcessor / DocFieldProcessorPerThread
--> DocFieldConsumer / DocFieldConsumerPerThread / DocFieldConsumerPerField
--> code: DocFieldConsumers / DocFieldConsumersPerThread / DocFieldConsumersPerField
--> code: DocInverter / DocInverterPerThread / DocInverterPerField
--> InvertedDocConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
--> code: TermsHash / TermsHashPerThread / TermsHashPerField
--> TermsHashConsumer / TermsHashConsumerPerThread / TermsHashConsumerPerField
--> code: FreqProxTermsWriter / FreqProxTermsWriterPerThread / FreqProxTermsWriterPerField
--> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerThread / TermVectorsTermsWriterPerField
--> InvertedDocEndConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
--> code: NormsWriter / NormsWriterPerThread / NormsWriterPerField
--> code: StoredFieldsWriter / StoredFieldsWriterPerThread / StoredFieldsWriterPerField
*/
// Build up indexing chain:
TermsHashConsumer termVectorsWriter = new TermVectorsTermsWriter(documentsWriter);
TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();
InvertedDocConsumer termsHash = new TermsHash(documentsWriter, true, freqProxWriter, new TermsHash(documentsWriter, false, termVectorsWriter, null));
NormsWriter normsWriter = new NormsWriter();
DocInverter docInverter = new DocInverter(termsHash, normsWriter);
return new DocFieldProcessor(documentsWriter, docInverter);
}
internal abstract DocConsumer GetChain(DocumentsWriter documentsWriter);
public StoredFieldsWriter(DocumentsWriter docWriter, FieldInfos fieldInfos)
{
InitBlock();
this.docWriter = docWriter;
this.fieldInfos = fieldInfos;
}
public StoredFieldsWriterPerThread AddThread(DocumentsWriter.DocState docState)
{
return new StoredFieldsWriterPerThread(docState, this);
}
protected internal virtual void RunFlushByRam(int numThreads, double maxRamMB, bool ensureNotStalled)
{
int numDocumentsToIndex = 10 + AtLeast(30);
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
Directory dir = NewDirectory();
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetRAMBufferSizeMB(maxRamMB);
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
Assert.IsFalse(flushPolicy.FlushOnDocCount);
Assert.IsFalse(flushPolicy.FlushOnDeleteTerms);
Assert.IsTrue(flushPolicy.FlushOnRAM);
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads);
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
AssertActiveBytesAfter(flushControl);
if (flushPolicy.HasMarkedPending)
{
Assert.IsTrue(maxRAMBytes < flushControl.peakActiveBytes);
}
if (ensureNotStalled)
{
Assert.IsFalse(docsWriter.flushControl.stallControl.WasStalled);
}
writer.Dispose();
Assert.AreEqual(0, flushControl.ActiveBytes);
dir.Dispose();
}
public PerDocBuffer(DocumentsWriter enclosingInstance)
{
this.enclosingInstance = enclosingInstance;
}
public TermVectorsTermsWriter(DocumentsWriter docWriter)
{
this.docWriter = docWriter;
}
/// <summary> Initialize the deleter: find all previous commits in
/// the Directory, incref the files they reference, call
/// the policy to let it delete commits. The incoming
/// segmentInfos must have been loaded from a commit point
/// and not yet modified. This will remove any files not
/// referenced by any of the commits.
/// </summary>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.TextWriter infoStream, DocumentsWriter docWriter)
{
this.docWriter = docWriter;
this.infoStream = infoStream;
if (infoStream != null)
{
Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy);
}
this.policy = policy;
this.directory = directory;
// First pass: walk the files and initialize our ref
// counts:
long currentGen = segmentInfos.GetGeneration();
IndexFileNameFilter filter = IndexFileNameFilter.GetFilter();
System.String[] files = directory.List();
if (files == null)
{
throw new System.IO.IOException("cannot read directory " + directory + ": list() returned null");
}
CommitPoint currentCommitPoint = null;
for (int i = 0; i < files.Length; i++)
{
System.String fileName = files[i];
if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
{
// Add this file to refCounts with initial count 0:
GetRefCount(fileName);
if (fileName.StartsWith(IndexFileNames.SEGMENTS))
{
// This is a commit (segments or segments_N), and
// it's valid (<= the max gen). Load it, then
// incref all files it refers to:
if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen)
{
if (infoStream != null)
{
Message("init: load commit \"" + fileName + "\"");
}
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, fileName);
}
catch (System.IO.FileNotFoundException e)
{
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
if (infoStream != null)
{
Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
}
sis = null;
}
if (sis != null)
{
CommitPoint commitPoint = new CommitPoint(this, sis);
if (sis.GetGeneration() == segmentInfos.GetGeneration())
{
currentCommitPoint = commitPoint;
}
commits.Add(commitPoint);
IncRef(sis, true);
}
}
}
}
}
if (currentCommitPoint == null)
{
// We did not in fact see the segments_N file
// corresponding to the segmentInfos that was passed
// in. Yet, it must exist, because our caller holds
// the write lock. This can happen when the directory
// listing was stale (eg when index accessed via NFS
// client with stale directory listing cache). So we
// try now to explicitly open this commit point:
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, segmentInfos.GetCurrentSegmentFileName());
}
catch (System.IO.IOException e)
{
throw new CorruptIndexException("failed to locate current segments_N file");
}
if (infoStream != null)
{
Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName());
}
currentCommitPoint = new CommitPoint(this, sis);
commits.Add(currentCommitPoint);
IncRef(sis, true);
}
// We keep commits list in sorted order (oldest to newest):
commits.Sort();
// Now delete anything with ref count at 0. These are
// presumably abandoned files eg due to crash of
// IndexWriter.
System.Collections.IEnumerator it = refCounts.Keys.GetEnumerator();
while (it.MoveNext())
{
System.String fileName = (System.String)it.Current;
RefCount rc = (RefCount)refCounts[fileName];
if (0 == rc.count)
{
if (infoStream != null)
{
Message("init: removing unreferenced file \"" + fileName + "\"");
}
DeleteFile(fileName);
}
}
// Finally, give policy a chance to remove things on
// startup:
policy.OnInit(commits);
// It's OK for the onInit to remove the current commit
// point; we just have to checkpoint our in-memory
// SegmentInfos to protect those files that it uses:
if (currentCommitPoint.deleted)
{
Checkpoint(segmentInfos, false);
}
DeleteCommits();
}
public IntBlockPool(DocumentsWriter docWriter, bool trackAllocations)
{
InitBlock();
this.docWriter = docWriter;
this.trackAllocations = trackAllocations;
}
public TermVectorsTermsWriter(DocumentsWriter docWriter)
{
InitBlock();
this.docWriter = docWriter;
}
public ByteBlockAllocator(DocumentsWriter enclosingInstance)
{
InitBlock(enclosingInstance);
}
public virtual void TestRandom()
{
int numThreads = 1 + Random.Next(8);
int numDocumentsToIndex = 50 + AtLeast(70);
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
iwc.SetFlushPolicy(flushPolicy);
int numDWPT = 1 + Random.Next(8);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, true);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due");
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
if (flushPolicy.FlushOnRAM && !flushPolicy.FlushOnDocCount && !flushPolicy.FlushOnDeleteTerms)
{
long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
if (flushPolicy.HasMarkedPending)
{
assertTrue("max: " + maxRAMBytes + " " + flushControl.peakActiveBytes, maxRAMBytes <= flushControl.peakActiveBytes);
}
}
AssertActiveBytesAfter(flushControl);
writer.Commit();
Assert.AreEqual(0, flushControl.ActiveBytes);
IndexReader r = DirectoryReader.Open(dir);
Assert.AreEqual(numDocumentsToIndex, r.NumDocs);
Assert.AreEqual(numDocumentsToIndex, r.MaxDoc);
if (!flushPolicy.FlushOnRAM)
{
assertFalse("never stall if we don't flush on RAM", docsWriter.flushControl.stallControl.WasStalled);
assertFalse("never block if we don't flush on RAM", docsWriter.flushControl.stallControl.HasBlocked);
}
r.Dispose();
writer.Dispose();
dir.Dispose();
}
public WaitQueue(DocumentsWriter enclosingInstance)
{
InitBlock(enclosingInstance);
waiting = new DocWriter[10];
}
public StoredFieldsWriter(DocumentsWriter docWriter)
{
this.docWriter = docWriter;
}
/// <summary> Initialize the deleter: find all previous commits in
/// the Directory, incref the files they reference, call
/// the policy to let it delete commits. This will remove
/// any files not referenced by any of the commits.
/// </summary>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.StreamWriter infoStream, DocumentsWriter docWriter, System.Collections.Generic.Dictionary<string, string> synced)
{
this.docWriter = docWriter;
this.infoStream = infoStream;
this.synced = synced;
if (infoStream != null)
{
Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy);
}
this.policy = policy;
this.directory = directory;
// First pass: walk the files and initialize our ref
// counts:
long currentGen = segmentInfos.GetGeneration();
IndexFileNameFilter filter = IndexFileNameFilter.GetFilter();
System.String[] files = directory.ListAll();
CommitPoint currentCommitPoint = null;
for (int i = 0; i < files.Length; i++)
{
System.String fileName = files[i];
if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
{
// Add this file to refCounts with initial count 0:
GetRefCount(fileName);
if (fileName.StartsWith(IndexFileNames.SEGMENTS))
{
// This is a commit (segments or segments_N), and
// it's valid (<= the max gen). Load it, then
// incref all files it refers to:
if (infoStream != null)
{
Message("init: load commit \"" + fileName + "\"");
}
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, fileName);
}
catch (System.IO.FileNotFoundException e)
{
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
if (infoStream != null)
{
Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
}
sis = null;
}
catch (System.IO.IOException e)
{
if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen)
{
throw e;
}
else
{
// Most likely we are opening an index that
// has an aborted "future" commit, so suppress
// exc in this case
sis = null;
}
}
if (sis != null)
{
CommitPoint commitPoint = new CommitPoint(this,commitsToDelete, directory, sis);
if (sis.GetGeneration() == segmentInfos.GetGeneration())
{
currentCommitPoint = commitPoint;
}
commits.Add(commitPoint);
IncRef(sis, true);
if (lastSegmentInfos == null || sis.GetGeneration() > lastSegmentInfos.GetGeneration())
{
lastSegmentInfos = sis;
}
}
}
}
}
if (currentCommitPoint == null)
{
// We did not in fact see the segments_N file
// corresponding to the segmentInfos that was passed
// in. Yet, it must exist, because our caller holds
// the write lock. This can happen when the directory
// listing was stale (eg when index accessed via NFS
// client with stale directory listing cache). So we
// try now to explicitly open this commit point:
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, segmentInfos.GetCurrentSegmentFileName());
}
catch (System.IO.IOException e)
{
throw new CorruptIndexException("failed to locate current segments_N file");
}
if (infoStream != null)
Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName());
currentCommitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
commits.Add(currentCommitPoint);
IncRef(sis, true);
}
// We keep commits list in sorted order (oldest to newest):
commits.Sort();
// Now delete anything with ref count at 0. These are
// presumably abandoned files eg due to crash of
// IndexWriter.
System.Collections.Generic.IEnumerator<System.Collections.Generic.KeyValuePair<System.String, RefCount>> it = refCounts.GetEnumerator();
while (it.MoveNext())
{
System.String fileName = (System.String) it.Current.Key;
RefCount rc = (RefCount) refCounts[fileName];
if (0 == rc.count)
{
if (infoStream != null)
{
Message("init: removing unreferenced file \"" + fileName + "\"");
}
DeleteFile(fileName);
}
}
// Finally, give policy a chance to remove things on
// startup:
policy.OnInit(commits);
// Always protect the incoming segmentInfos since
// sometime it may not be the most recent commit
Checkpoint(segmentInfos, false);
startingCommitDeleted = currentCommitPoint.IsDeleted();
DeleteCommits();
}
private void CloseInternal(bool waitForMerges)
{
docWriter.PauseAllThreads();
try
{
if (infoStream != null)
Message("now flush at close");
docWriter.Close();
// Only allow a new merge to be triggered if we are
// going to wait for merges:
if (!hitOOM)
{
Flush(waitForMerges, true, true);
}
if (waitForMerges)
// Give merge scheduler last chance to run, in case
// any pending merges are waiting:
mergeScheduler.Merge(this);
mergePolicy.Close();
FinishMerges(waitForMerges);
stopMerges = true;
mergeScheduler.Close();
if (infoStream != null)
Message("now call final commit()");
if (!hitOOM)
{
Commit(0);
}
if (infoStream != null)
Message("at close: " + SegString());
lock (this)
{
readerPool.Close();
docWriter = null;
deleter.Close();
}
if (closeDir)
directory.Close();
if (writeLock != null)
{
writeLock.Release(); // release write lock
writeLock = null;
}
lock (this)
{
closed = true;
}
}
catch (System.OutOfMemoryException oom)
{
HandleOOM(oom, "closeInternal");
}
finally
{
lock (this)
{
closing = false;
System.Threading.Monitor.PulseAll(this);
if (!closed)
{
if (docWriter != null)
docWriter.ResumeAllThreads();
if (infoStream != null)
Message("hit exception while closing");
}
}
}
}
// you cannot subclass this without being in o.a.l.index package anyway, so // the class is already pkg-private... fix me: see LUCENE-4013 public abstract ThreadState GetAndLock(Thread requestingThread, DocumentsWriter documentsWriter); // LUCENENET NOTE: Made public rather than internal
internal readonly Codec Codec; // for writing new segments
/// <summary>
/// Constructs a new IndexWriter per the settings given in <code>conf</code>.
/// If you want to make "live" changes to this writer instance, use
/// <seealso cref="#getConfig()"/>.
///
/// <p>
/// <b>NOTE:</b> after ths writer is created, the given configuration instance
/// cannot be passed to another writer. If you intend to do so, you should
/// <seealso cref="IndexWriterConfig#clone() clone"/> it beforehand.
/// </summary>
/// <param name="d">
/// the index directory. The index is either created or appended
/// according <code>conf.getOpenMode()</code>. </param>
/// <param name="conf">
/// the configuration settings according to which IndexWriter should
/// be initialized. </param>
/// <exception cref="IOException">
/// if the directory cannot be read/written to, or if it does not
/// exist and <code>conf.getOpenMode()</code> is
/// <code>OpenMode.APPEND</code> or if there is any other low-level
/// IO error </exception>
public IndexWriter(Directory d, IndexWriterConfig conf)
{
/*if (!InstanceFieldsInitialized)
{
InitializeInstanceFields();
InstanceFieldsInitialized = true;
}*/
readerPool = new ReaderPool(this);
conf.SetIndexWriter(this); // prevent reuse by other instances
Config_Renamed = new LiveIndexWriterConfig(conf);
directory = d;
analyzer = Config_Renamed.Analyzer;
infoStream = Config_Renamed.InfoStream;
mergePolicy = Config_Renamed.MergePolicy;
mergePolicy.IndexWriter = this;
mergeScheduler = Config_Renamed.MergeScheduler;
Codec = Config_Renamed.Codec;
BufferedUpdatesStream = new BufferedUpdatesStream(infoStream);
PoolReaders = Config_Renamed.ReaderPooling;
WriteLock = directory.MakeLock(WRITE_LOCK_NAME);
if (!WriteLock.Obtain(Config_Renamed.WriteLockTimeout)) // obtain write lock
{
throw new LockObtainFailedException("Index locked for write: " + WriteLock);
}
bool success = false;
try
{
OpenMode_e? mode = Config_Renamed.OpenMode;
bool create;
if (mode == OpenMode_e.CREATE)
{
create = true;
}
else if (mode == OpenMode_e.APPEND)
{
create = false;
}
else
{
// CREATE_OR_APPEND - create only if an index does not exist
create = !DirectoryReader.IndexExists(directory);
}
// If index is too old, reading the segments will throw
// IndexFormatTooOldException.
segmentInfos = new SegmentInfos();
bool initialIndexExists = true;
if (create)
{
// Try to read first. this is to allow create
// against an index that's currently open for
// searching. In this case we write the next
// segments_N file with no segments:
try
{
segmentInfos.Read(directory);
segmentInfos.Clear();
}
catch (IOException)
{
// Likely this means it's a fresh directory
initialIndexExists = false;
}
// Record that we have a change (zero out all
// segments) pending:
Changed();
}
else
{
segmentInfos.Read(directory);
IndexCommit commit = Config_Renamed.IndexCommit;
if (commit != null)
{
// Swap out all segments, but, keep metadata in
// SegmentInfos, like version & generation, to
// preserve write-once. this is important if
// readers are open against the future commit
// points.
if (commit.Directory != directory)
{
throw new System.ArgumentException("IndexCommit's directory doesn't match my directory");
}
SegmentInfos oldInfos = new SegmentInfos();
oldInfos.Read(directory, commit.SegmentsFileName);
segmentInfos.Replace(oldInfos);
Changed();
if (infoStream.IsEnabled("IW"))
{
infoStream.Message("IW", "init: loaded commit \"" + commit.SegmentsFileName + "\"");
}
}
}
RollbackSegments = segmentInfos.CreateBackupSegmentInfos();
// start with previous field numbers, but new FieldInfos
GlobalFieldNumberMap = FieldNumberMap;
Config_Renamed.FlushPolicy.Init(Config_Renamed);
DocWriter = new DocumentsWriter(this, Config_Renamed, directory);
eventQueue = DocWriter.EventQueue();
// Default deleter (for backwards compatibility) is
// KeepOnlyLastCommitDeleter:
lock (this)
{
Deleter = new IndexFileDeleter(directory, Config_Renamed.DelPolicy, segmentInfos, infoStream, this, initialIndexExists);
}
if (Deleter.StartingCommitDeleted)
{
// Deletion policy deleted the "head" commit point.
// We have to mark ourself as changed so that if we
// are closed w/o any further changes we write a new
// segments_N file.
Changed();
}
if (infoStream.IsEnabled("IW"))
{
infoStream.Message("IW", "init: create=" + create);
MessageState();
}
success = true;
}
finally
{
if (!success)
{
if (infoStream.IsEnabled("IW"))
{
infoStream.Message("IW", "init: hit exception on init; releasing write lock");
}
WriteLock.Release();
IOUtils.CloseWhileHandlingException(WriteLock);
WriteLock = null;
}
}
}
internal ByteBlockAllocator(DocumentsWriter enclosing_instance)
{
this.enclosing_instance = enclosing_instance;
}
public IntBlockPool(DocumentsWriter docWriter, bool trackAllocations)
{
InitBlock();
this.docWriter = docWriter;
this.trackAllocations = trackAllocations;
}
public StoredFieldsWriter(DocumentsWriter docWriter, FieldInfos fieldInfos)
{
InitBlock();
this.docWriter = docWriter;
this.fieldInfos = fieldInfos;
}
public WaitQueue(DocumentsWriter enclosing_instance)
{
this.enclosing_instance = enclosing_instance;
waiting = new DocWriter[10];
}
public TermVectorsTermsWriter(DocumentsWriter docWriter)
{
InitBlock();
this.docWriter = docWriter;
}
private void Init(Directory d, Analyzer a, bool create, bool closeDir, IndexDeletionPolicy deletionPolicy, bool autoCommit, int maxFieldLength, IndexingChain indexingChain, IndexCommit commit)
{
this.closeDir = closeDir;
directory = d;
analyzer = a;
SetMessageID(defaultInfoStream);
this.maxFieldLength = maxFieldLength;
if (indexingChain == null)
indexingChain = DocumentsWriter.DefaultIndexingChain;
if (create)
{
// Clear the write lock in case it's leftover:
directory.ClearLock(WRITE_LOCK_NAME);
}
Lock writeLock = directory.MakeLock(WRITE_LOCK_NAME);
if (!writeLock.Obtain(writeLockTimeout))
// obtain write lock
{
throw new LockObtainFailedException("Index locked for write: " + writeLock);
}
this.writeLock = writeLock; // save it
bool success = false;
try
{
if (create)
{
// Try to read first. This is to allow create
// against an index that's currently open for
// searching. In this case we write the next
// segments_N file with no segments:
bool doCommit;
try
{
segmentInfos.Read(directory);
segmentInfos.Clear();
doCommit = false;
}
catch (System.IO.IOException e)
{
// Likely this means it's a fresh directory
doCommit = true;
}
if (autoCommit || doCommit)
{
// Always commit if autoCommit=true, else only
// commit if there is no segments file in this dir
// already.
segmentInfos.Commit(directory);
SupportClass.CollectionsHelper.AddAllIfNotContains(synced, segmentInfos.Files(directory, true));
}
else
{
// Record that we have a change (zero out all
// segments) pending:
changeCount++;
}
}
else
{
segmentInfos.Read(directory);
if (commit != null)
{
// Swap out all segments, but, keep metadata in
// SegmentInfos, like version & generation, to
// preserve write-once. This is important if
// readers are open against the future commit
// points.
if (commit.GetDirectory() != directory)
throw new System.ArgumentException("IndexCommit's directory doesn't match my directory");
SegmentInfos oldInfos = new SegmentInfos();
oldInfos.Read(directory, commit.GetSegmentsFileName());
segmentInfos.Replace(oldInfos);
changeCount++;
if (infoStream != null)
Message("init: loaded commit \"" + commit.GetSegmentsFileName() + "\"");
}
// We assume that this segments_N was previously
// properly sync'd:
SupportClass.CollectionsHelper.AddAllIfNotContains(synced, segmentInfos.Files(directory, true));
}
this.autoCommit = autoCommit;
SetRollbackSegmentInfos(segmentInfos);
docWriter = new DocumentsWriter(directory, this, indexingChain);
docWriter.SetInfoStream(infoStream);
docWriter.SetMaxFieldLength(maxFieldLength);
// Default deleter (for backwards compatibility) is
// KeepOnlyLastCommitDeleter:
deleter = new IndexFileDeleter(directory, deletionPolicy == null?new KeepOnlyLastCommitDeletionPolicy():deletionPolicy, segmentInfos, infoStream, docWriter,synced);
if (deleter.startingCommitDeleted)
// Deletion policy deleted the "head" commit point.
// We have to mark ourself as changed so that if we
// are closed w/o any further changes we write a new
// segments_N file.
changeCount++;
PushMaxBufferedDocs();
if (infoStream != null)
{
Message("init: create=" + create);
MessageState();
}
success = true;
}
finally
{
if (!success)
{
if (infoStream != null)
{
Message("init: hit exception on init; releasing write lock");
}
try
{
writeLock.Release();
}
catch (Exception t)
{
// don't mask the original exception
}
writeLock = null;
}
}
}
public CharBlockPool(DocumentsWriter docWriter)
{
InitBlock();
this.docWriter = docWriter;
}
public DocFieldProcessor(DocumentsWriter docWriter, DocFieldConsumer consumer)
{
this.docWriter = docWriter;
this.consumer = consumer;
consumer.setFieldInfos(fieldInfos);
}
