public DocFieldProcessor(DocumentsWriter docWriter, DocFieldConsumer consumer) { this.docWriter = docWriter; this.consumer = consumer; consumer.SetFieldInfos(fieldInfos); fieldsWriter = new StoredFieldsWriter(docWriter, fieldInfos); }
internal override void closeDocStore(DocumentsWriter.FlushState state) { lock (this) { int inc = state.numDocsInStore - lastDocID; if (inc > 0) { initFieldsWriter(); fill(state.numDocsInStore - docWriter.GetDocStoreOffset()); } if (fieldsWriter != null) { fieldsWriter.Close(); fieldsWriter = null; lastDocID = 0; System.Diagnostics.Debug.Assert(state.docStoreSegmentName != null); string fdtFile = state.docStoreSegmentName + "." + IndexFileNames.FIELDS_EXTENSION; string fdxFile = state.docStoreSegmentName + "." + IndexFileNames.FIELDS_INDEX_EXTENSION; state.flushedFiles[fdtFile] = fdtFile; state.flushedFiles[fdxFile] = fdxFile; state.docWriter.RemoveOpenFile(fdtFile); state.docWriter.RemoveOpenFile(fdxFile); if (4 + state.numDocsInStore * 8 != state.directory.FileLength(fdxFile)) throw new System.SystemException("after flush: fdx size mismatch: " + state.numDocsInStore + " docs vs " + state.directory.FileLength(fdxFile) + " length in bytes of " + fdxFile); } } }
public override ThreadState GetAndLock(Thread requestingThread, DocumentsWriter documentsWriter) { ThreadState threadState = null; if (ActiveThreadState == 0) { lock (this) { if (ActiveThreadState == 0) { threadState = States[0] = NewThreadState(); return threadState; } } } Debug.Assert(ActiveThreadState > 0); for (int i = 0; i < MaxRetry; i++) { int ord = Random.Next(ActiveThreadState); lock (this) { threadState = States[ord]; Debug.Assert(threadState != null); } if (threadState.TryLock()) { return threadState; } if (Random.Next(20) == 0) { break; } } /* * only try to create a new threadstate if we can not lock the randomly * selected state. this is important since some tests rely on a single * threadstate in the single threaded case. Eventually it would be nice if * we would not have this limitation but for now we just make sure we only * allocate one threadstate if indexing is single threaded */ lock (this) { ThreadState newThreadState = NewThreadState(); if (newThreadState != null) // did we get a new state? { threadState = States[ActiveThreadState - 1] = newThreadState; //Debug.Assert(threadState.HeldByCurrentThread); return threadState; } // if no new state is available lock the random one } Debug.Assert(threadState != null); threadState.@Lock(); return threadState; }
internal int numThreads = 1; // Number of threads that share this instance #endregion Fields #region Constructors public DocumentsWriterThreadState(DocumentsWriter docWriter) { this.docWriter = docWriter; docState = new DocumentsWriter.DocState(); docState.maxFieldLength = docWriter.maxFieldLength; docState.infoStream = docWriter.infoStream; docState.similarity = docWriter.similarity; docState.docWriter = docWriter; consumer = docWriter.consumer.AddThread(this); }
public DocumentsWriterThreadState(DocumentsWriter docWriter) { this.docWriter = docWriter; docState = new DocumentsWriter.DocState(); docState.maxFieldLength = docWriter.maxFieldLength; docState.infoStream = docWriter.infoStream; docState.similarity = docWriter.similarity; docState.docWriter = docWriter; docState.allowMinus1Position = docWriter.writer.GetAllowMinus1Position(); consumer = docWriter.consumer.AddThread(this); }
internal DocumentsWriterFlushControl(DocumentsWriter documentsWriter, LiveIndexWriterConfig config, BufferedUpdatesStream bufferedUpdatesStream) { this.InfoStream_Renamed = config.InfoStream; this.StallControl = new DocumentsWriterStallControl(); this.PerThreadPool = documentsWriter.PerThreadPool; this.FlushPolicy = documentsWriter.FlushPolicy; this.Config = config; this.HardMaxBytesPerDWPT = config.RAMPerThreadHardLimitMB * 1024 * 1024; this.DocumentsWriter = documentsWriter; this.BufferedUpdatesStream = bufferedUpdatesStream; }
internal DocumentsWriterFlushControl(DocumentsWriter documentsWriter, LiveIndexWriterConfig config, BufferedUpdatesStream bufferedUpdatesStream) { this.infoStream = config.InfoStream; this.stallControl = new DocumentsWriterStallControl(); this.perThreadPool = documentsWriter.perThreadPool; this.flushPolicy = documentsWriter.flushPolicy; this.config = config; this.hardMaxBytesPerDWPT = config.RAMPerThreadHardLimitMB * 1024 * 1024; this.documentsWriter = documentsWriter; this.bufferedUpdatesStream = bufferedUpdatesStream; }
public SegmentWriteState(DocumentsWriter docWriter, Directory directory, System.String segmentName, System.String docStoreSegmentName, int numDocs, int numDocsInStore, int termIndexInterval) { this.docWriter = docWriter; this.directory = directory; this.segmentName = segmentName; this.docStoreSegmentName = docStoreSegmentName; this.numDocs = numDocs; this.numDocsInStore = numDocsInStore; this.termIndexInterval = termIndexInterval; flushedFiles = new Support.Set <string>(); }
internal override void closeDocStore(DocumentsWriter.FlushState state) { try { one.closeDocStore(state); } finally { two.closeDocStore(state); } }
internal int numThreads = 1; // Number of threads that share this instance #endregion Fields #region Constructors public DocumentsWriterThreadState(DocumentsWriter docWriter) { this.docWriter = docWriter; docState = new DocumentsWriter.DocState(); docState.maxFieldLength = docWriter.maxFieldLength; docState.infoStream = docWriter.infoStream; docState.similarity = docWriter.similarity; docState.docWriter = docWriter; docState.allowMinus1Position = docWriter.writer.GetAllowMinus1Position(); consumer = docWriter.consumer.AddThread(this); }
public SegmentWriteState(DocumentsWriter docWriter, Directory directory, System.String segmentName, System.String docStoreSegmentName, int numDocs, int numDocsInStore, int termIndexInterval) { this.docWriter = docWriter; this.directory = directory; this.segmentName = segmentName; this.docStoreSegmentName = docStoreSegmentName; this.numDocs = numDocs; this.numDocsInStore = numDocsInStore; this.termIndexInterval = termIndexInterval; flushedFiles = new Support.Set<string>(); }
public virtual void TestStallControl() { int[] numThreads = new int[] { 4 + Random.Next(8), 1 }; int numDocumentsToIndex = 50 + Random.Next(50); for (int i = 0; i < numThreads.Length; i++) { AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex); MockDirectoryWrapper dir = NewMockDirectory(); // mock a very slow harddisk sometimes here so that flushing is very slow dir.Throttling = Throttling.SOMETIMES; IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)); iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH); FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy(); iwc.SetFlushPolicy(flushPolicy); DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numThreads[i] == 1 ? 1 : 2); iwc.SetIndexerThreadPool(threadPool); // with such a small ram buffer we should be stalled quiet quickly iwc.SetRAMBufferSizeMB(0.25); IndexWriter writer = new IndexWriter(dir, iwc); IndexThread[] threads = new IndexThread[numThreads[i]]; for (int x = 0; x < threads.Length; x++) { threads[x] = new IndexThread(this, numDocs, numThreads[i], writer, LineDocFile, false); threads[x].Start(); } for (int x = 0; x < threads.Length; x++) { threads[x].Join(); } DocumentsWriter docsWriter = writer.DocsWriter; Assert.IsNotNull(docsWriter); DocumentsWriterFlushControl flushControl = docsWriter.flushControl; Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due"); Assert.AreEqual(numDocumentsToIndex, writer.NumDocs); Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc); if (numThreads[i] == 1) { assertFalse("single thread must not block numThreads: " + numThreads[i], docsWriter.flushControl.stallControl.HasBlocked); } if (docsWriter.flushControl.peakNetBytes > (2d * iwc.RAMBufferSizeMB * 1024d * 1024d)) { Assert.IsTrue(docsWriter.flushControl.stallControl.WasStalled); } AssertActiveBytesAfter(flushControl); writer.Dispose(true); dir.Dispose(); } }
public virtual void TestFlushDocCount() { // LUCENENET specific - disable the test if asserts are not enabled AssumeTrue("This test requires asserts to be enabled.", Debugging.AssertsEnabled); int[] numThreads = new int[] { 2 + AtLeast(1), 1 }; for (int i = 0; i < numThreads.Length; i++) { int numDocumentsToIndex = 50 + AtLeast(30); AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex); Directory dir = NewDirectory(); MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy(); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetFlushPolicy(flushPolicy); int numDWPT = 1 + AtLeast(2); DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numDWPT); iwc.SetIndexerThreadPool(threadPool); iwc.SetMaxBufferedDocs(2 + AtLeast(10)); iwc.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH); IndexWriter writer = new IndexWriter(dir, iwc); flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy; Assert.IsTrue(flushPolicy.FlushOnDocCount); Assert.IsFalse(flushPolicy.FlushOnDeleteTerms); Assert.IsFalse(flushPolicy.FlushOnRAM); DocumentsWriter docsWriter = writer.DocsWriter; Assert.IsNotNull(docsWriter); DocumentsWriterFlushControl flushControl = docsWriter.flushControl; Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init"); IndexThread[] threads = new IndexThread[numThreads[i]]; for (int x = 0; x < threads.Length; x++) { threads[x] = new IndexThread(numDocs, writer, lineDocFile, false); threads[x].Start(); } for (int x = 0; x < threads.Length; x++) { threads[x].Join(); } Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads[i]); Assert.AreEqual(numDocumentsToIndex, writer.NumDocs); Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc); Assert.IsTrue(flushPolicy.peakDocCountWithoutFlush <= iwc.MaxBufferedDocs, "peak bytes without flush exceeded watermark"); AssertActiveBytesAfter(flushControl); writer.Dispose(); Assert.AreEqual(0, flushControl.ActiveBytes); dir.Dispose(); } }
//internal override void flush(/*IDictionary*/ IDictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> threadsAndFields, DocumentsWriter.FlushState state) internal override void flush(/*IDictionary*/ IDictionary<object, ICollection<object>> threadsAndFields, DocumentsWriter.FlushState state) { //IDictionary oneThreadsAndFields = new Dictionary(); //IDictionary twoThreadsAndFields = new Dictionary(); //IDictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> oneThreadsAndFields = new Dictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>>(); //IDictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> twoThreadsAndFields = new Dictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>>(); IDictionary<object, ICollection<object>> oneThreadsAndFields = new Dictionary<object, ICollection<object>>(); IDictionary<object, ICollection<object>> twoThreadsAndFields = new Dictionary<object, ICollection<object>>(); //IEnumerator it = threadsAndFields.GetEnumerator(); //IEnumerator<KeyValuePair<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>>> it = threadsAndFields.GetEnumerator(); IEnumerator<KeyValuePair<object, ICollection<object>>> it = threadsAndFields.GetEnumerator(); while (it.MoveNext()) { //KeyValuePair entry = it.Current; //KeyValuePair<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> entry = it.Current; KeyValuePair<object, ICollection<object>> entry = it.Current; //DocFieldConsumersPerThread perThread = (DocFieldConsumersPerThread)entry.Key; DocFieldConsumersPerThread perThread = (DocFieldConsumersPerThread)entry.Key; //ICollection fields = (ICollection)entry.Value; //ICollection<DocFieldConsumerPerField> fields = entry.Value; ICollection<object> fields = entry.Value; //IEnumerator fieldsIt = fields.GetEnumerator(); //IEnumerator<DocFieldConsumerPerField> fieldsIt = fields.GetEnumerator(); IEnumerator<object> fieldsIt = fields.GetEnumerator(); //IDictionary oneFields = new Dictionary(); //IDictionary twoFields = new Dictionary(); //IDictionary<DocFieldConsumerPerField, DocFieldConsumerPerField> oneFields = new Dictionary<DocFieldConsumerPerField, DocFieldConsumerPerField>(); //IDictionary<DocFieldConsumerPerField, DocFieldConsumerPerField> twoFields = new Dictionary<DocFieldConsumerPerField, DocFieldConsumerPerField>(); IDictionary<object, object> oneFields = new Dictionary<object, object>(); IDictionary<object, object> twoFields = new Dictionary<object, object>(); while (fieldsIt.MoveNext()) { DocFieldConsumersPerField perField = (DocFieldConsumersPerField)fieldsIt.Current; oneFields[perField.one] = perField.one; twoFields[perField.two] = perField.two; } oneThreadsAndFields[perThread.one] = oneFields.Keys; twoThreadsAndFields[perThread.two] = twoFields.Keys; } one.flush(oneThreadsAndFields, state); two.flush(twoThreadsAndFields, state); }
public TermsHash(DocumentsWriter docWriter, bool trackAllocations, TermsHashConsumer consumer, TermsHash nextTermsHash) { this.docWriter = docWriter; this.consumer = consumer; this.nextTermsHash = nextTermsHash; this.trackAllocations = trackAllocations; // Why + 4*POINTER_NUM_BYTE below? // +1: Posting is referenced by postingsFreeList array // +3: Posting is referenced by hash, which // targets 25-50% fill factor; approximate this // as 3X # pointers bytesPerPosting = consumer.BytesPerPosting() + 4 * DocumentsWriter.POINTER_NUM_BYTE; postingsFreeChunk = (int)(DocumentsWriter.BYTE_BLOCK_SIZE / bytesPerPosting); }
public TermsHash(DocumentsWriter docWriter, bool trackAllocations, TermsHashConsumer consumer, TermsHash nextTermsHash) { this.docWriter = docWriter; this.consumer = consumer; this.nextTermsHash = nextTermsHash; this.trackAllocations = trackAllocations; // Why + 4*POINTER_NUM_BYTE below? // +1: Posting is referenced by postingsFreeList array // +3: Posting is referenced by hash, which // targets 25-50% fill factor; approximate this // as 3X # pointers bytesPerPosting = consumer.BytesPerPosting() + 4 * DocumentsWriter.POINTER_NUM_BYTE; postingsFreeChunk = (int) (DocumentsWriter.BYTE_BLOCK_SIZE / bytesPerPosting); }
internal override void Flush(ICollection<object> threads, DocumentsWriter.FlushState state) { IDictionary<object, ICollection<object>> childThreadsAndFields = new Dictionary<object, ICollection<object>>(); IEnumerator<object> it = threads.GetEnumerator(); while (it.MoveNext()) { DocFieldProcessorPerThread perThread = (DocFieldProcessorPerThread)it.Current; childThreadsAndFields[perThread.consumer] = perThread.Fields(); perThread.trimFields(state); } consumer.flush(childThreadsAndFields, state); // Important to save after asking consumer to flush so // consumer can alter the FieldInfo* if necessary. EG, // FreqProxTermsWriter does this with // FieldInfo.storePayload. fieldInfos.Write(state.directory, state.segmentName + ".fnm"); }
public override ThreadState GetAndLock(Thread requestingThread, DocumentsWriter documentsWriter) { if (threadBindings.TryGetValue(requestingThread, out ThreadState threadState) && threadState.TryLock()) { return(threadState); } ThreadState minThreadState = null; /* TODO -- another thread could lock the minThreadState we just got while * we should somehow prevent this. */ // Find the state that has minimum number of threads waiting minThreadState = MinContendedThreadState(); if (minThreadState == null || minThreadState.HasQueuedThreads) { ThreadState newState = NewThreadState(); // state is already locked if non-null if (newState != null) { if (Debugging.AssertsEnabled) { Debugging.Assert(newState.IsHeldByCurrentThread); } threadBindings[requestingThread] = newState; return(newState); } else if (minThreadState == null) { /* * no new threadState available we just take the minContented one * this must return a valid thread state since we accessed the * synced context in newThreadState() above. */ minThreadState = MinContendedThreadState(); } } if (Debugging.AssertsEnabled) { Debugging.Assert(minThreadState != null, () => "ThreadState is null"); } minThreadState.@Lock(); return(minThreadState); }
//internal override void flush(IDictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> threadsAndFields, DocumentsWriter.FlushState state) internal override void flush(IDictionary<object, ICollection<object>> threadsAndFields, DocumentsWriter.FlushState state) { //IDictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> childThreadsAndFields = new Dictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>>(); //IDictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> endChildThreadsAndFields = new Dictionary<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>>(); IDictionary<object, ICollection<object>> childThreadsAndFields = new Dictionary<object, ICollection<object>>(); IDictionary<object, ICollection<object>> endChildThreadsAndFields = new Dictionary<object, ICollection<object>>(); //IEnumerator<KeyValuePair<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>>> it = threadsAndFields.GetEnumerator(); IEnumerator<KeyValuePair<object, ICollection<object>>> it = threadsAndFields.GetEnumerator(); while (it.MoveNext()) { //KeyValuePair<DocFieldConsumerPerThread, ICollection<DocFieldConsumerPerField>> entry = it.Current; KeyValuePair<object, ICollection<object>> entry = it.Current; DocInverterPerThread perThread = (DocInverterPerThread)entry.Key; //ICollection<DocFieldConsumerPerField> fields = entry.Value; ICollection<object> fields = entry.Value; //IEnumerator<DocFieldConsumerPerField> fieldsIt = fields.GetEnumerator(); IEnumerator<object> fieldsIt = fields.GetEnumerator(); //IDictionary<DocFieldConsumerPerField, DocFieldConsumerPerField> childFields = new Dictionary<DocFieldConsumerPerField, DocFieldConsumerPerField>(); //IDictionary<DocFieldConsumerPerField, DocFieldConsumerPerField> endChildFields = new Dictionary<DocFieldConsumerPerField, DocFieldConsumerPerField>(); IDictionary<object, object> childFields = new Dictionary<object, object>(); IDictionary<object, object> endChildFields = new Dictionary<object, object>(); while (fieldsIt.MoveNext()) { DocInverterPerField perField = (DocInverterPerField)fieldsIt.Current; childFields[perField.consumer] = perField.consumer; endChildFields[perField.endConsumer] = perField.endConsumer; } childThreadsAndFields[perThread.consumer] = childFields.Keys; // create new collection to provide for deletions in NormsWriter endChildThreadsAndFields[perThread.endConsumer] = new List<object>(endChildFields.Keys); } consumer.flush(childThreadsAndFields, state); endConsumer.flush(endChildThreadsAndFields, state); }
public override ThreadState GetAndLock(Thread requestingThread, DocumentsWriter documentsWriter) { ThreadState threadState; ThreadBindings.TryGetValue(requestingThread, out threadState); if (threadState != null && threadState.TryLock()) { return threadState; } ThreadState minThreadState = null; /* TODO -- another thread could lock the minThreadState we just got while we should somehow prevent this. */ // Find the state that has minimum number of threads waiting minThreadState = MinContendedThreadState(); if (minThreadState == null || minThreadState.HasQueuedThreads) { ThreadState newState = NewThreadState(); // state is already locked if non-null if (newState != null) { //Debug.Assert(newState.HeldByCurrentThread); ThreadBindings[requestingThread] = newState; return newState; } else if (minThreadState == null) { /* * no new threadState available we just take the minContented one * this must return a valid thread state since we accessed the * synced context in newThreadState() above. */ minThreadState = MinContendedThreadState(); } } Debug.Assert(minThreadState != null, "ThreadState is null"); minThreadState.@Lock(); return minThreadState; }
public CharBlockPool(DocumentsWriter docWriter) { InitBlock(); this.docWriter = docWriter; }
// you cannot subclass this without being in o.a.l.index package anyway, so // the class is already pkg-private... fix me: see LUCENE-4013 public abstract ThreadState GetAndLock(Thread requestingThread, DocumentsWriter documentsWriter);
/// <summary> Initialize the deleter: find all previous commits in /// the Directory, incref the files they reference, call /// the policy to let it delete commits. This will remove /// any files not referenced by any of the commits. /// </summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.StreamWriter infoStream, DocumentsWriter docWriter, HashSet <string> synced) { this.docWriter = docWriter; this.infoStream = infoStream; this.synced = synced; if (infoStream != null) { Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy); } this.policy = policy; this.directory = directory; // First pass: walk the files and initialize our ref // counts: long currentGen = segmentInfos.Generation; IndexFileNameFilter filter = IndexFileNameFilter.Filter; System.String[] files = directory.ListAll(); CommitPoint currentCommitPoint = null; for (int i = 0; i < files.Length; i++) { System.String fileName = files[i]; if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN)) { // Add this file to refCounts with initial count 0: GetRefCount(fileName); if (fileName.StartsWith(IndexFileNames.SEGMENTS)) { // This is a commit (segments or segments_N), and // it's valid (<= the max gen). Load it, then // incref all files it refers to: if (infoStream != null) { Message("init: load commit \"" + fileName + "\""); } SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, fileName); } catch (System.IO.FileNotFoundException) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist if (infoStream != null) { Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point"); } sis = null; } catch (System.IO.IOException) { if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen) { throw; } else { // Most likely we are opening an index that // has an aborted "future" commit, so suppress // exc in this case sis = null; } } if (sis != null) { CommitPoint commitPoint = new CommitPoint(this, commitsToDelete, directory, sis); if (sis.Generation == segmentInfos.Generation) { currentCommitPoint = commitPoint; } commits.Add(commitPoint); IncRef(sis, true); if (lastSegmentInfos == null || sis.Generation > lastSegmentInfos.Generation) { lastSegmentInfos = sis; } } } } } if (currentCommitPoint == null) { // We did not in fact see the segments_N file // corresponding to the segmentInfos that was passed // in. Yet, it must exist, because our caller holds // the write lock. This can happen when the directory // listing was stale (eg when index accessed via NFS // client with stale directory listing cache). So we // try now to explicitly open this commit point: SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, segmentInfos.GetCurrentSegmentFileName()); } catch (System.IO.IOException) { throw new CorruptIndexException("failed to locate current segments_N file"); } if (infoStream != null) { Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName()); } currentCommitPoint = new CommitPoint(this, commitsToDelete, directory, sis); commits.Add(currentCommitPoint); IncRef(sis, true); } // We keep commits list in sorted order (oldest to newest): commits.Sort(); // Now delete anything with ref count at 0. These are // presumably abandoned files eg due to crash of // IndexWriter. foreach (KeyValuePair <string, RefCount> entry in refCounts) { string fileName = entry.Key; RefCount rc = refCounts[fileName]; if (0 == rc.count) { if (infoStream != null) { Message("init: removing unreferenced file \"" + fileName + "\""); } DeleteFile(fileName); } } // Finally, give policy a chance to remove things on // startup: policy.OnInit(commits); // Always protect the incoming segmentInfos since // sometime it may not be the most recent commit Checkpoint(segmentInfos, false); startingCommitDeleted = currentCommitPoint.IsDeleted; DeleteCommits(); }
public StoredFieldsWriterPerThread(DocumentsWriter.DocState docState, StoredFieldsWriter storedFieldsWriter) { this.storedFieldsWriter = storedFieldsWriter; this.docState = docState; localFieldsWriter = new FieldsWriter((IndexOutput) null, (IndexOutput) null, storedFieldsWriter.fieldInfos); }
public override ThreadState GetAndLock(Thread requestingThread, DocumentsWriter documentsWriter) { ThreadState threadState = null; if (NumThreadStatesActive == 0) { lock (this) { if (NumThreadStatesActive == 0) { threadState = states[0] = NewThreadState(); return(threadState); } } } if (Debugging.AssertsEnabled) { Debugging.Assert(NumThreadStatesActive > 0); } for (int i = 0; i < maxRetry; i++) { int ord = random.Next(NumThreadStatesActive); lock (this) { threadState = states[ord]; if (Debugging.AssertsEnabled) { Debugging.Assert(threadState != null); } } if (threadState.TryLock()) { return(threadState); } if (random.Next(20) == 0) { break; } } /* * only try to create a new threadstate if we can not lock the randomly * selected state. this is important since some tests rely on a single * threadstate in the single threaded case. Eventually it would be nice if * we would not have this limitation but for now we just make sure we only * allocate one threadstate if indexing is single threaded */ lock (this) { ThreadState newThreadState = NewThreadState(); if (newThreadState != null) // did we get a new state? { threadState = states[NumThreadStatesActive - 1] = newThreadState; //if (Debugging.AssertsEnabled) Debugging.Assert(threadState.HeldByCurrentThread); return(threadState); } // if no new state is available lock the random one } if (Debugging.AssertsEnabled) { Debugging.Assert(threadState != null); } threadState.@Lock(); return(threadState); }
private void InitBlock(DocumentsWriter enclosingInstance) { this.enclosingInstance = enclosingInstance; }
internal override DocConsumer GetChain(DocumentsWriter documentsWriter) { /* This is the current indexing chain: DocConsumer / DocConsumerPerThread --> code: DocFieldProcessor / DocFieldProcessorPerThread --> DocFieldConsumer / DocFieldConsumerPerThread / DocFieldConsumerPerField --> code: DocFieldConsumers / DocFieldConsumersPerThread / DocFieldConsumersPerField --> code: DocInverter / DocInverterPerThread / DocInverterPerField --> InvertedDocConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField --> code: TermsHash / TermsHashPerThread / TermsHashPerField --> TermsHashConsumer / TermsHashConsumerPerThread / TermsHashConsumerPerField --> code: FreqProxTermsWriter / FreqProxTermsWriterPerThread / FreqProxTermsWriterPerField --> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerThread / TermVectorsTermsWriterPerField --> InvertedDocEndConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField --> code: NormsWriter / NormsWriterPerThread / NormsWriterPerField --> code: StoredFieldsWriter / StoredFieldsWriterPerThread / StoredFieldsWriterPerField */ // Build up indexing chain: TermsHashConsumer termVectorsWriter = new TermVectorsTermsWriter(documentsWriter); TermsHashConsumer freqProxWriter = new FreqProxTermsWriter(); InvertedDocConsumer termsHash = new TermsHash(documentsWriter, true, freqProxWriter, new TermsHash(documentsWriter, false, termVectorsWriter, null)); NormsWriter normsWriter = new NormsWriter(); DocInverter docInverter = new DocInverter(termsHash, normsWriter); return new DocFieldProcessor(documentsWriter, docInverter); }
internal abstract DocConsumer GetChain(DocumentsWriter documentsWriter);
public StoredFieldsWriter(DocumentsWriter docWriter, FieldInfos fieldInfos) { InitBlock(); this.docWriter = docWriter; this.fieldInfos = fieldInfos; }
public StoredFieldsWriterPerThread AddThread(DocumentsWriter.DocState docState) { return new StoredFieldsWriterPerThread(docState, this); }
protected internal virtual void RunFlushByRam(int numThreads, double maxRamMB, bool ensureNotStalled) { int numDocumentsToIndex = 10 + AtLeast(30); AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex); Directory dir = NewDirectory(); MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy(); MockAnalyzer analyzer = new MockAnalyzer(Random); analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetFlushPolicy(flushPolicy); int numDWPT = 1 + AtLeast(2); DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT); iwc.SetIndexerThreadPool(threadPool); iwc.SetRAMBufferSizeMB(maxRamMB); iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH); IndexWriter writer = new IndexWriter(dir, iwc); flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy; Assert.IsFalse(flushPolicy.FlushOnDocCount); Assert.IsFalse(flushPolicy.FlushOnDeleteTerms); Assert.IsTrue(flushPolicy.FlushOnRAM); DocumentsWriter docsWriter = writer.DocsWriter; Assert.IsNotNull(docsWriter); DocumentsWriterFlushControl flushControl = docsWriter.flushControl; Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init"); IndexThread[] threads = new IndexThread[numThreads]; for (int x = 0; x < threads.Length; x++) { threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, false); threads[x].Start(); } for (int x = 0; x < threads.Length; x++) { threads[x].Join(); } long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0); Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads); Assert.AreEqual(numDocumentsToIndex, writer.NumDocs); Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc); Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark"); AssertActiveBytesAfter(flushControl); if (flushPolicy.HasMarkedPending) { Assert.IsTrue(maxRAMBytes < flushControl.peakActiveBytes); } if (ensureNotStalled) { Assert.IsFalse(docsWriter.flushControl.stallControl.WasStalled); } writer.Dispose(); Assert.AreEqual(0, flushControl.ActiveBytes); dir.Dispose(); }
public PerDocBuffer(DocumentsWriter enclosingInstance) { this.enclosingInstance = enclosingInstance; }
public TermVectorsTermsWriter(DocumentsWriter docWriter) { this.docWriter = docWriter; }
/// <summary> Initialize the deleter: find all previous commits in /// the Directory, incref the files they reference, call /// the policy to let it delete commits. The incoming /// segmentInfos must have been loaded from a commit point /// and not yet modified. This will remove any files not /// referenced by any of the commits. /// </summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.TextWriter infoStream, DocumentsWriter docWriter) { this.docWriter = docWriter; this.infoStream = infoStream; if (infoStream != null) { Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy); } this.policy = policy; this.directory = directory; // First pass: walk the files and initialize our ref // counts: long currentGen = segmentInfos.GetGeneration(); IndexFileNameFilter filter = IndexFileNameFilter.GetFilter(); System.String[] files = directory.List(); if (files == null) { throw new System.IO.IOException("cannot read directory " + directory + ": list() returned null"); } CommitPoint currentCommitPoint = null; for (int i = 0; i < files.Length; i++) { System.String fileName = files[i]; if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN)) { // Add this file to refCounts with initial count 0: GetRefCount(fileName); if (fileName.StartsWith(IndexFileNames.SEGMENTS)) { // This is a commit (segments or segments_N), and // it's valid (<= the max gen). Load it, then // incref all files it refers to: if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen) { if (infoStream != null) { Message("init: load commit \"" + fileName + "\""); } SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, fileName); } catch (System.IO.FileNotFoundException e) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist if (infoStream != null) { Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point"); } sis = null; } if (sis != null) { CommitPoint commitPoint = new CommitPoint(this, sis); if (sis.GetGeneration() == segmentInfos.GetGeneration()) { currentCommitPoint = commitPoint; } commits.Add(commitPoint); IncRef(sis, true); } } } } } if (currentCommitPoint == null) { // We did not in fact see the segments_N file // corresponding to the segmentInfos that was passed // in. Yet, it must exist, because our caller holds // the write lock. This can happen when the directory // listing was stale (eg when index accessed via NFS // client with stale directory listing cache). So we // try now to explicitly open this commit point: SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, segmentInfos.GetCurrentSegmentFileName()); } catch (System.IO.IOException e) { throw new CorruptIndexException("failed to locate current segments_N file"); } if (infoStream != null) { Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName()); } currentCommitPoint = new CommitPoint(this, sis); commits.Add(currentCommitPoint); IncRef(sis, true); } // We keep commits list in sorted order (oldest to newest): commits.Sort(); // Now delete anything with ref count at 0. These are // presumably abandoned files eg due to crash of // IndexWriter. System.Collections.IEnumerator it = refCounts.Keys.GetEnumerator(); while (it.MoveNext()) { System.String fileName = (System.String)it.Current; RefCount rc = (RefCount)refCounts[fileName]; if (0 == rc.count) { if (infoStream != null) { Message("init: removing unreferenced file \"" + fileName + "\""); } DeleteFile(fileName); } } // Finally, give policy a chance to remove things on // startup: policy.OnInit(commits); // It's OK for the onInit to remove the current commit // point; we just have to checkpoint our in-memory // SegmentInfos to protect those files that it uses: if (currentCommitPoint.deleted) { Checkpoint(segmentInfos, false); } DeleteCommits(); }
public IntBlockPool(DocumentsWriter docWriter, bool trackAllocations) { InitBlock(); this.docWriter = docWriter; this.trackAllocations = trackAllocations; }
public TermVectorsTermsWriter(DocumentsWriter docWriter) { InitBlock(); this.docWriter = docWriter; }
public ByteBlockAllocator(DocumentsWriter enclosingInstance) { InitBlock(enclosingInstance); }
public virtual void TestRandom() { int numThreads = 1 + Random.Next(8); int numDocumentsToIndex = 50 + AtLeast(70); AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex); Directory dir = NewDirectory(); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)); MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy(); iwc.SetFlushPolicy(flushPolicy); int numDWPT = 1 + Random.Next(8); DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT); iwc.SetIndexerThreadPool(threadPool); IndexWriter writer = new IndexWriter(dir, iwc); flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy; DocumentsWriter docsWriter = writer.DocsWriter; Assert.IsNotNull(docsWriter); DocumentsWriterFlushControl flushControl = docsWriter.flushControl; Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init"); IndexThread[] threads = new IndexThread[numThreads]; for (int x = 0; x < threads.Length; x++) { threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, true); threads[x].Start(); } for (int x = 0; x < threads.Length; x++) { threads[x].Join(); } Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due"); Assert.AreEqual(numDocumentsToIndex, writer.NumDocs); Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc); if (flushPolicy.FlushOnRAM && !flushPolicy.FlushOnDocCount && !flushPolicy.FlushOnDeleteTerms) { long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0); Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark"); if (flushPolicy.HasMarkedPending) { assertTrue("max: " + maxRAMBytes + " " + flushControl.peakActiveBytes, maxRAMBytes <= flushControl.peakActiveBytes); } } AssertActiveBytesAfter(flushControl); writer.Commit(); Assert.AreEqual(0, flushControl.ActiveBytes); IndexReader r = DirectoryReader.Open(dir); Assert.AreEqual(numDocumentsToIndex, r.NumDocs); Assert.AreEqual(numDocumentsToIndex, r.MaxDoc); if (!flushPolicy.FlushOnRAM) { assertFalse("never stall if we don't flush on RAM", docsWriter.flushControl.stallControl.WasStalled); assertFalse("never block if we don't flush on RAM", docsWriter.flushControl.stallControl.HasBlocked); } r.Dispose(); writer.Dispose(); dir.Dispose(); }
public WaitQueue(DocumentsWriter enclosingInstance) { InitBlock(enclosingInstance); waiting = new DocWriter[10]; }
public StoredFieldsWriter(DocumentsWriter docWriter) { this.docWriter = docWriter; }
/// <summary> Initialize the deleter: find all previous commits in /// the Directory, incref the files they reference, call /// the policy to let it delete commits. This will remove /// any files not referenced by any of the commits. /// </summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.StreamWriter infoStream, DocumentsWriter docWriter, System.Collections.Generic.Dictionary<string, string> synced) { this.docWriter = docWriter; this.infoStream = infoStream; this.synced = synced; if (infoStream != null) { Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy); } this.policy = policy; this.directory = directory; // First pass: walk the files and initialize our ref // counts: long currentGen = segmentInfos.GetGeneration(); IndexFileNameFilter filter = IndexFileNameFilter.GetFilter(); System.String[] files = directory.ListAll(); CommitPoint currentCommitPoint = null; for (int i = 0; i < files.Length; i++) { System.String fileName = files[i]; if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN)) { // Add this file to refCounts with initial count 0: GetRefCount(fileName); if (fileName.StartsWith(IndexFileNames.SEGMENTS)) { // This is a commit (segments or segments_N), and // it's valid (<= the max gen). Load it, then // incref all files it refers to: if (infoStream != null) { Message("init: load commit \"" + fileName + "\""); } SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, fileName); } catch (System.IO.FileNotFoundException e) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist if (infoStream != null) { Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point"); } sis = null; } catch (System.IO.IOException e) { if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen) { throw e; } else { // Most likely we are opening an index that // has an aborted "future" commit, so suppress // exc in this case sis = null; } } if (sis != null) { CommitPoint commitPoint = new CommitPoint(this,commitsToDelete, directory, sis); if (sis.GetGeneration() == segmentInfos.GetGeneration()) { currentCommitPoint = commitPoint; } commits.Add(commitPoint); IncRef(sis, true); if (lastSegmentInfos == null || sis.GetGeneration() > lastSegmentInfos.GetGeneration()) { lastSegmentInfos = sis; } } } } } if (currentCommitPoint == null) { // We did not in fact see the segments_N file // corresponding to the segmentInfos that was passed // in. Yet, it must exist, because our caller holds // the write lock. This can happen when the directory // listing was stale (eg when index accessed via NFS // client with stale directory listing cache). So we // try now to explicitly open this commit point: SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, segmentInfos.GetCurrentSegmentFileName()); } catch (System.IO.IOException e) { throw new CorruptIndexException("failed to locate current segments_N file"); } if (infoStream != null) Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName()); currentCommitPoint = new CommitPoint(this, commitsToDelete, directory, sis); commits.Add(currentCommitPoint); IncRef(sis, true); } // We keep commits list in sorted order (oldest to newest): commits.Sort(); // Now delete anything with ref count at 0. These are // presumably abandoned files eg due to crash of // IndexWriter. System.Collections.Generic.IEnumerator<System.Collections.Generic.KeyValuePair<System.String, RefCount>> it = refCounts.GetEnumerator(); while (it.MoveNext()) { System.String fileName = (System.String) it.Current.Key; RefCount rc = (RefCount) refCounts[fileName]; if (0 == rc.count) { if (infoStream != null) { Message("init: removing unreferenced file \"" + fileName + "\""); } DeleteFile(fileName); } } // Finally, give policy a chance to remove things on // startup: policy.OnInit(commits); // Always protect the incoming segmentInfos since // sometime it may not be the most recent commit Checkpoint(segmentInfos, false); startingCommitDeleted = currentCommitPoint.IsDeleted(); DeleteCommits(); }
private void CloseInternal(bool waitForMerges) { docWriter.PauseAllThreads(); try { if (infoStream != null) Message("now flush at close"); docWriter.Close(); // Only allow a new merge to be triggered if we are // going to wait for merges: if (!hitOOM) { Flush(waitForMerges, true, true); } if (waitForMerges) // Give merge scheduler last chance to run, in case // any pending merges are waiting: mergeScheduler.Merge(this); mergePolicy.Close(); FinishMerges(waitForMerges); stopMerges = true; mergeScheduler.Close(); if (infoStream != null) Message("now call final commit()"); if (!hitOOM) { Commit(0); } if (infoStream != null) Message("at close: " + SegString()); lock (this) { readerPool.Close(); docWriter = null; deleter.Close(); } if (closeDir) directory.Close(); if (writeLock != null) { writeLock.Release(); // release write lock writeLock = null; } lock (this) { closed = true; } } catch (System.OutOfMemoryException oom) { HandleOOM(oom, "closeInternal"); } finally { lock (this) { closing = false; System.Threading.Monitor.PulseAll(this); if (!closed) { if (docWriter != null) docWriter.ResumeAllThreads(); if (infoStream != null) Message("hit exception while closing"); } } } }
// you cannot subclass this without being in o.a.l.index package anyway, so // the class is already pkg-private... fix me: see LUCENE-4013 public abstract ThreadState GetAndLock(Thread requestingThread, DocumentsWriter documentsWriter); // LUCENENET NOTE: Made public rather than internal
internal readonly Codec Codec; // for writing new segments /// <summary> /// Constructs a new IndexWriter per the settings given in <code>conf</code>. /// If you want to make "live" changes to this writer instance, use /// <seealso cref="#getConfig()"/>. /// /// <p> /// <b>NOTE:</b> after ths writer is created, the given configuration instance /// cannot be passed to another writer. If you intend to do so, you should /// <seealso cref="IndexWriterConfig#clone() clone"/> it beforehand. /// </summary> /// <param name="d"> /// the index directory. The index is either created or appended /// according <code>conf.getOpenMode()</code>. </param> /// <param name="conf"> /// the configuration settings according to which IndexWriter should /// be initialized. </param> /// <exception cref="IOException"> /// if the directory cannot be read/written to, or if it does not /// exist and <code>conf.getOpenMode()</code> is /// <code>OpenMode.APPEND</code> or if there is any other low-level /// IO error </exception> public IndexWriter(Directory d, IndexWriterConfig conf) { /*if (!InstanceFieldsInitialized) { InitializeInstanceFields(); InstanceFieldsInitialized = true; }*/ readerPool = new ReaderPool(this); conf.SetIndexWriter(this); // prevent reuse by other instances Config_Renamed = new LiveIndexWriterConfig(conf); directory = d; analyzer = Config_Renamed.Analyzer; infoStream = Config_Renamed.InfoStream; mergePolicy = Config_Renamed.MergePolicy; mergePolicy.IndexWriter = this; mergeScheduler = Config_Renamed.MergeScheduler; Codec = Config_Renamed.Codec; BufferedUpdatesStream = new BufferedUpdatesStream(infoStream); PoolReaders = Config_Renamed.ReaderPooling; WriteLock = directory.MakeLock(WRITE_LOCK_NAME); if (!WriteLock.Obtain(Config_Renamed.WriteLockTimeout)) // obtain write lock { throw new LockObtainFailedException("Index locked for write: " + WriteLock); } bool success = false; try { OpenMode_e? mode = Config_Renamed.OpenMode; bool create; if (mode == OpenMode_e.CREATE) { create = true; } else if (mode == OpenMode_e.APPEND) { create = false; } else { // CREATE_OR_APPEND - create only if an index does not exist create = !DirectoryReader.IndexExists(directory); } // If index is too old, reading the segments will throw // IndexFormatTooOldException. segmentInfos = new SegmentInfos(); bool initialIndexExists = true; if (create) { // Try to read first. this is to allow create // against an index that's currently open for // searching. In this case we write the next // segments_N file with no segments: try { segmentInfos.Read(directory); segmentInfos.Clear(); } catch (IOException) { // Likely this means it's a fresh directory initialIndexExists = false; } // Record that we have a change (zero out all // segments) pending: Changed(); } else { segmentInfos.Read(directory); IndexCommit commit = Config_Renamed.IndexCommit; if (commit != null) { // Swap out all segments, but, keep metadata in // SegmentInfos, like version & generation, to // preserve write-once. this is important if // readers are open against the future commit // points. if (commit.Directory != directory) { throw new System.ArgumentException("IndexCommit's directory doesn't match my directory"); } SegmentInfos oldInfos = new SegmentInfos(); oldInfos.Read(directory, commit.SegmentsFileName); segmentInfos.Replace(oldInfos); Changed(); if (infoStream.IsEnabled("IW")) { infoStream.Message("IW", "init: loaded commit \"" + commit.SegmentsFileName + "\""); } } } RollbackSegments = segmentInfos.CreateBackupSegmentInfos(); // start with previous field numbers, but new FieldInfos GlobalFieldNumberMap = FieldNumberMap; Config_Renamed.FlushPolicy.Init(Config_Renamed); DocWriter = new DocumentsWriter(this, Config_Renamed, directory); eventQueue = DocWriter.EventQueue(); // Default deleter (for backwards compatibility) is // KeepOnlyLastCommitDeleter: lock (this) { Deleter = new IndexFileDeleter(directory, Config_Renamed.DelPolicy, segmentInfos, infoStream, this, initialIndexExists); } if (Deleter.StartingCommitDeleted) { // Deletion policy deleted the "head" commit point. // We have to mark ourself as changed so that if we // are closed w/o any further changes we write a new // segments_N file. Changed(); } if (infoStream.IsEnabled("IW")) { infoStream.Message("IW", "init: create=" + create); MessageState(); } success = true; } finally { if (!success) { if (infoStream.IsEnabled("IW")) { infoStream.Message("IW", "init: hit exception on init; releasing write lock"); } WriteLock.Release(); IOUtils.CloseWhileHandlingException(WriteLock); WriteLock = null; } } }
internal ByteBlockAllocator(DocumentsWriter enclosing_instance) { this.enclosing_instance = enclosing_instance; }
public WaitQueue(DocumentsWriter enclosing_instance) { this.enclosing_instance = enclosing_instance; waiting = new DocWriter[10]; }
private void Init(Directory d, Analyzer a, bool create, bool closeDir, IndexDeletionPolicy deletionPolicy, bool autoCommit, int maxFieldLength, IndexingChain indexingChain, IndexCommit commit) { this.closeDir = closeDir; directory = d; analyzer = a; SetMessageID(defaultInfoStream); this.maxFieldLength = maxFieldLength; if (indexingChain == null) indexingChain = DocumentsWriter.DefaultIndexingChain; if (create) { // Clear the write lock in case it's leftover: directory.ClearLock(WRITE_LOCK_NAME); } Lock writeLock = directory.MakeLock(WRITE_LOCK_NAME); if (!writeLock.Obtain(writeLockTimeout)) // obtain write lock { throw new LockObtainFailedException("Index locked for write: " + writeLock); } this.writeLock = writeLock; // save it bool success = false; try { if (create) { // Try to read first. This is to allow create // against an index that's currently open for // searching. In this case we write the next // segments_N file with no segments: bool doCommit; try { segmentInfos.Read(directory); segmentInfos.Clear(); doCommit = false; } catch (System.IO.IOException e) { // Likely this means it's a fresh directory doCommit = true; } if (autoCommit || doCommit) { // Always commit if autoCommit=true, else only // commit if there is no segments file in this dir // already. segmentInfos.Commit(directory); SupportClass.CollectionsHelper.AddAllIfNotContains(synced, segmentInfos.Files(directory, true)); } else { // Record that we have a change (zero out all // segments) pending: changeCount++; } } else { segmentInfos.Read(directory); if (commit != null) { // Swap out all segments, but, keep metadata in // SegmentInfos, like version & generation, to // preserve write-once. This is important if // readers are open against the future commit // points. if (commit.GetDirectory() != directory) throw new System.ArgumentException("IndexCommit's directory doesn't match my directory"); SegmentInfos oldInfos = new SegmentInfos(); oldInfos.Read(directory, commit.GetSegmentsFileName()); segmentInfos.Replace(oldInfos); changeCount++; if (infoStream != null) Message("init: loaded commit \"" + commit.GetSegmentsFileName() + "\""); } // We assume that this segments_N was previously // properly sync'd: SupportClass.CollectionsHelper.AddAllIfNotContains(synced, segmentInfos.Files(directory, true)); } this.autoCommit = autoCommit; SetRollbackSegmentInfos(segmentInfos); docWriter = new DocumentsWriter(directory, this, indexingChain); docWriter.SetInfoStream(infoStream); docWriter.SetMaxFieldLength(maxFieldLength); // Default deleter (for backwards compatibility) is // KeepOnlyLastCommitDeleter: deleter = new IndexFileDeleter(directory, deletionPolicy == null?new KeepOnlyLastCommitDeletionPolicy():deletionPolicy, segmentInfos, infoStream, docWriter,synced); if (deleter.startingCommitDeleted) // Deletion policy deleted the "head" commit point. // We have to mark ourself as changed so that if we // are closed w/o any further changes we write a new // segments_N file. changeCount++; PushMaxBufferedDocs(); if (infoStream != null) { Message("init: create=" + create); MessageState(); } success = true; } finally { if (!success) { if (infoStream != null) { Message("init: hit exception on init; releasing write lock"); } try { writeLock.Release(); } catch (Exception t) { // don't mask the original exception } writeLock = null; } } }
public DocFieldProcessor(DocumentsWriter docWriter, DocFieldConsumer consumer) { this.docWriter = docWriter; this.consumer = consumer; consumer.setFieldInfos(fieldInfos); }