public DocFieldProcessor(DocumentsWriter docWriter, DocFieldConsumer consumer)
{
this.docWriter = docWriter;
this.consumer = consumer;
consumer.SetFieldInfos(fieldInfos);
fieldsWriter = new StoredFieldsWriter(docWriter, fieldInfos);
}
public SegmentWriteState(DocumentsWriter docWriter, Directory directory, System.String segmentName, System.String docStoreSegmentName, int numDocs, int numDocsInStore, int termIndexInterval)
{
this.docWriter = docWriter;
this.directory = directory;
this.segmentName = segmentName;
this.docStoreSegmentName = docStoreSegmentName;
this.numDocs = numDocs;
this.numDocsInStore = numDocsInStore;
this.termIndexInterval = termIndexInterval;
flushedFiles = new System.Collections.Generic.HashSet<string>();
}
public TermsHash(DocumentsWriter docWriter, bool trackAllocations, TermsHashConsumer consumer, TermsHash nextTermsHash)
{
this.docWriter = docWriter;
this.consumer = consumer;
this.nextTermsHash = nextTermsHash;
this.trackAllocations = trackAllocations;
// Why + 4*POINTER_NUM_BYTE below?
// +1: Posting is referenced by postingsFreeList array
// +3: Posting is referenced by hash, which
// targets 25-50% fill factor; approximate this
// as 3X # pointers
bytesPerPosting = consumer.BytesPerPosting() + 4 * DocumentsWriter.POINTER_NUM_BYTE;
postingsFreeChunk = (int) (DocumentsWriter.BYTE_BLOCK_SIZE / bytesPerPosting);
}
private void Init(Directory d, Analyzer a, IndexDeletionPolicy deletionPolicy, int maxFieldLength, DocumentsWriter.IndexingChain indexingChain, IndexCommit commit)
{
if (IndexReader.IndexExists(d))
{
Init(d, a, false, deletionPolicy, maxFieldLength, indexingChain, commit);
}
else
{
Init(d, a, true, deletionPolicy, maxFieldLength, indexingChain, commit);
}
}
/// <summary> Expert: constructs an IndexWriter with a custom <see cref="IndexDeletionPolicy" />
/// and <see cref="DocumentsWriter.IndexingChain" />,
/// for the index in <c>d</c>.
/// Text will be analyzed with <c>a</c>. If
/// <c>create</c> is true, then a new, empty index
/// will be created in <c>d</c>, replacing the index
/// already there, if any.
///
/// </summary>
/// <param name="d">the index directory
/// </param>
/// <param name="a">the analyzer to use
/// </param>
/// <param name="create"><c>true</c> to create the index or overwrite
/// the existing one; <c>false</c> to append to the existing
/// index
/// </param>
/// <param name="deletionPolicy">see <a href="#deletionPolicy">above</a>
/// </param>
/// <param name="mfl">whether or not to limit field lengths, value is in number of terms/tokens. See <see cref="Lucene.Net.Index.IndexWriter.MaxFieldLength" />.
/// </param>
/// <param name="indexingChain">the <see cref="DocConsumer" /> chain to be used to
/// process documents
/// </param>
/// <param name="commit">which commit to open
/// </param>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> LockObtainFailedException if another writer </throws>
/// <summary> has this index open (<c>write.lock</c> could not
/// be obtained)
/// </summary>
/// <throws> IOException if the directory cannot be read/written to, or </throws>
/// <summary> if it does not exist and <c>create</c> is
/// <c>false</c> or if there is any other low-level
/// IO error
/// </summary>
internal IndexWriter(Directory d, Analyzer a, bool create, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl, DocumentsWriter.IndexingChain indexingChain, IndexCommit commit)
{
InitBlock();
Init(d, a, create, deletionPolicy, mfl.Limit, indexingChain, commit);
}
internal override DocConsumer GetChain(DocumentsWriter documentsWriter)
{
/*
This is the current indexing chain:
DocConsumer / DocConsumerPerThread
--> code: DocFieldProcessor / DocFieldProcessorPerThread
--> DocFieldConsumer / DocFieldConsumerPerThread / DocFieldConsumerPerField
--> code: DocFieldConsumers / DocFieldConsumersPerThread / DocFieldConsumersPerField
--> code: DocInverter / DocInverterPerThread / DocInverterPerField
--> InvertedDocConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
--> code: TermsHash / TermsHashPerThread / TermsHashPerField
--> TermsHashConsumer / TermsHashConsumerPerThread / TermsHashConsumerPerField
--> code: FreqProxTermsWriter / FreqProxTermsWriterPerThread / FreqProxTermsWriterPerField
--> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerThread / TermVectorsTermsWriterPerField
--> InvertedDocEndConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
--> code: NormsWriter / NormsWriterPerThread / NormsWriterPerField
--> code: StoredFieldsWriter / StoredFieldsWriterPerThread / StoredFieldsWriterPerField
*/
// Build up indexing chain:
TermsHashConsumer termVectorsWriter = new TermVectorsTermsWriter(documentsWriter);
TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();
InvertedDocConsumer termsHash = new TermsHash(documentsWriter, true, freqProxWriter, new TermsHash(documentsWriter, false, termVectorsWriter, null));
NormsWriter normsWriter = new NormsWriter();
DocInverter docInverter = new DocInverter(termsHash, normsWriter);
return new DocFieldProcessor(documentsWriter, docInverter);
}
internal abstract DocConsumer GetChain(DocumentsWriter documentsWriter);
public PerDocBuffer(DocumentsWriter enclosingInstance)
{
this.enclosingInstance = enclosingInstance;
}
public StoredFieldsWriterPerThread AddThread(DocumentsWriter.DocState docState)
{
return new StoredFieldsWriterPerThread(docState, this);
}
private void InitBlock(DocumentsWriter enclosingInstance)
{
this.enclosingInstance = enclosingInstance;
}
public ByteBlockAllocator(DocumentsWriter enclosingInstance, int blockSize)
{
this.blockSize = blockSize;
InitBlock(enclosingInstance);
}
public TermVectorsTermsWriter(DocumentsWriter docWriter)
{
InitBlock();
this.docWriter = docWriter;
}
public CharBlockPool(DocumentsWriter docWriter)
{
InitBlock();
this.docWriter = docWriter;
}
private void Init(Directory d, Analyzer a, bool create, IndexDeletionPolicy deletionPolicy, int maxFieldLength, DocumentsWriter.IndexingChain indexingChain, IndexCommit commit)
{
directory = d;
analyzer = a;
SetMessageID(defaultInfoStream);
this.maxFieldLength = maxFieldLength;
if (indexingChain == null)
indexingChain = DocumentsWriter.DefaultIndexingChain;
if (create)
{
// Clear the write lock in case it's leftover:
directory.ClearLock(WRITE_LOCK_NAME);
}
Lock writeLock = directory.MakeLock(WRITE_LOCK_NAME);
if (!writeLock.Obtain(writeLockTimeout))
// obtain write lock
{
throw new LockObtainFailedException("Index locked for write: " + writeLock);
}
this.writeLock = writeLock; // save it
bool success = false;
try
{
if (create)
{
// Try to read first. This is to allow create
// against an index that's currently open for
// searching. In this case we write the next
// segments_N file with no segments:
bool doCommit;
try
{
segmentInfos.Read(directory);
segmentInfos.Clear();
doCommit = false;
}
catch (System.IO.IOException)
{
// Likely this means it's a fresh directory
doCommit = true;
}
if (doCommit)
{
// Only commit if there is no segments file
// in this dir already.
segmentInfos.Commit(directory);
synced.UnionWith(segmentInfos.Files(directory, true));
}
else
{
// Record that we have a change (zero out all
// segments) pending:
changeCount++;
}
}
else
{
segmentInfos.Read(directory);
if (commit != null)
{
// Swap out all segments, but, keep metadata in
// SegmentInfos, like version & generation, to
// preserve write-once. This is important if
// readers are open against the future commit
// points.
if (commit.Directory != directory)
throw new System.ArgumentException("IndexCommit's directory doesn't match my directory");
SegmentInfos oldInfos = new SegmentInfos();
oldInfos.Read(directory, commit.SegmentsFileName);
segmentInfos.Replace(oldInfos);
changeCount++;
if (infoStream != null)
Message("init: loaded commit \"" + commit.SegmentsFileName + "\"");
}
// We assume that this segments_N was previously
// properly sync'd:
synced.UnionWith(segmentInfos.Files(directory, true));
}
SetRollbackSegmentInfos(segmentInfos);
docWriter = new DocumentsWriter(directory, this, indexingChain);
docWriter.SetInfoStream(infoStream);
docWriter.SetMaxFieldLength(maxFieldLength);
// Default deleter (for backwards compatibility) is
// KeepOnlyLastCommitDeleter:
deleter = new IndexFileDeleter(directory, deletionPolicy == null?new KeepOnlyLastCommitDeletionPolicy():deletionPolicy, segmentInfos, infoStream, docWriter, synced);
if (deleter.startingCommitDeleted)
// Deletion policy deleted the "head" commit point.
// We have to mark ourself as changed so that if we
// are closed w/o any further changes we write a new
// segments_N file.
changeCount++;
PushMaxBufferedDocs();
if (infoStream != null)
{
Message("init: create=" + create);
MessageState();
}
success = true;
}
finally
{
if (!success)
{
if (infoStream != null)
{
Message("init: hit exception on init; releasing write lock");
}
try
{
writeLock.Release();
}
catch (Exception)
{
// don't mask the original exception
}
writeLock = null;
}
}
}
public WaitQueue(DocumentsWriter enclosingInstance)
{
InitBlock(enclosingInstance);
waiting = new DocWriter[10];
}
private void CloseInternal(bool waitForMerges)
{
docWriter.PauseAllThreads();
try
{
if (infoStream != null)
Message("now flush at close");
docWriter.Dispose();
// Only allow a new merge to be triggered if we are
// going to wait for merges:
if (!hitOOM)
{
Flush(waitForMerges, true, true);
}
if (waitForMerges)
// Give merge scheduler last chance to run, in case
// any pending merges are waiting:
mergeScheduler.Merge(this);
mergePolicy.Close();
FinishMerges(waitForMerges);
stopMerges = true;
mergeScheduler.Close();
if (infoStream != null)
Message("now call final commit()");
if (!hitOOM)
{
Commit(0);
}
if (infoStream != null)
Message("at close: " + SegString());
lock (this)
{
readerPool.Dispose();
docWriter = null;
deleter.Dispose();
}
if (writeLock != null)
{
writeLock.Release(); // release write lock
writeLock = null;
}
lock (this)
{
closed = true;
}
}
catch (System.OutOfMemoryException oom)
{
HandleOOM(oom, "closeInternal");
}
finally
{
lock (this)
{
closing = false;
System.Threading.Monitor.PulseAll(this);
if (!closed)
{
if (docWriter != null)
docWriter.ResumeAllThreads();
if (infoStream != null)
Message("hit exception while closing");
}
}
}
}
public StoredFieldsWriter(DocumentsWriter docWriter, FieldInfos fieldInfos)
{
InitBlock();
this.docWriter = docWriter;
this.fieldInfos = fieldInfos;
}