public DocumentsWriterThreadState(DocumentsWriter docWriter) { this.docWriter = docWriter; docState = new DocumentsWriter.DocState(); docState.maxFieldLength = docWriter.maxFieldLength; docState.infoStream = docWriter.infoStream; docState.similarity = docWriter.similarity; docState.docWriter = docWriter; consumer = docWriter.consumer.AddThread(this); }
public DocumentsWriterThreadState(DocumentsWriter docWriter) { this.docWriter = docWriter; docState = new DocumentsWriter.DocState(); docState.maxFieldLength = docWriter.maxFieldLength; docState.infoStream = docWriter.infoStream; docState.similarity = docWriter.similarity; docState.docWriter = docWriter; docState.allowMinus1Position = docWriter.writer.GetAllowMinus1Position(); consumer = docWriter.consumer.AddThread(this); }
public SegmentWriteState(DocumentsWriter docWriter, Directory directory, System.String segmentName, System.String docStoreSegmentName, int numDocs, int numDocsInStore, int termIndexInterval) { this.docWriter = docWriter; this.directory = directory; this.segmentName = segmentName; this.docStoreSegmentName = docStoreSegmentName; this.numDocs = numDocs; this.numDocsInStore = numDocsInStore; this.termIndexInterval = termIndexInterval; flushedFiles = new System.Collections.Generic.HashSet <string>(); }
public TermsHash(DocumentsWriter docWriter, bool trackAllocations, TermsHashConsumer consumer, TermsHash nextTermsHash) { this.docWriter = docWriter; this.consumer = consumer; this.nextTermsHash = nextTermsHash; this.trackAllocations = trackAllocations; // Why + 4*POINTER_NUM_BYTE below? // +1: Posting is referenced by postingsFreeList array // +3: Posting is referenced by hash, which // targets 25-50% fill factor; approximate this // as 3X # pointers bytesPerPosting = consumer.BytesPerPosting() + 4 * DocumentsWriter.POINTER_NUM_BYTE; postingsFreeChunk = (int)(DocumentsWriter.BYTE_BLOCK_SIZE / bytesPerPosting); }
public async Task WriteAsync_ShouldPassNullToRetryChannelForInsertionWorkers_RetryWorkersCountIsZero() { // Arrange _workerFactoryMock.Setup(f => f.RunWorker(It.IsAny <ChannelWriter <ReplaceOneModel <BsonDocument> > >(), It.IsAny <ILogger>(), It.IsAny <CancellationToken>())) .ReturnsAsync(_fixture.Create <WorkerResult>()); _workerFactoryMock.Setup(f => f.RunRetryWorker(It.IsAny <ChannelReader <ReplaceOneModel <BsonDocument> > >(), It.IsAny <ILogger>(), It.IsAny <CancellationToken>())) .ReturnsAsync(_fixture.Create <WorkerResult>()); var writer = new DocumentsWriter(10, 0, _fixture.Create <string>(), _workerFactoryMock.Object, _loggerMock.Object); // Act await writer.WriteAsync(CancellationToken.None); // Assert _workerFactoryMock.Verify(f => f.RunWorker(null, It.IsAny <ILogger>(), It.IsAny <CancellationToken>()), Times.Exactly(10)); }
public async Task WriteAsync_ShouldStartExactNumberOfWorkers_NumberOfInsertionWorkersGreaterThanZero(int insertionWorkers, int retryWorkers) { // Arrange _workerFactoryMock.Setup(f => f.RunWorker(It.IsAny <ChannelWriter <ReplaceOneModel <BsonDocument> > >(), It.IsAny <ILogger>(), It.IsAny <CancellationToken>())) .ReturnsAsync(new WorkerResult(100, 0)); _workerFactoryMock.Setup(f => f.RunRetryWorker(It.IsAny <ChannelReader <ReplaceOneModel <BsonDocument> > >(), It.IsAny <ILogger>(), It.IsAny <CancellationToken>())) .ReturnsAsync(new WorkerResult(100, 0)); var writer = new DocumentsWriter(insertionWorkers, retryWorkers, _fixture.Create <string>(), _workerFactoryMock.Object, _loggerMock.Object); // Act await writer.WriteAsync(CancellationToken.None); // Assert _workerFactoryMock.Verify(f => f.RunWorker(It.IsAny <ChannelWriter <ReplaceOneModel <BsonDocument> > >(), It.IsAny <ILogger>(), It.IsAny <CancellationToken>()), Times.Exactly(insertionWorkers)); _workerFactoryMock.Verify(f => f.RunRetryWorker(It.IsAny <ChannelReader <ReplaceOneModel <BsonDocument> > >(), It.IsAny <ILogger>(), It.IsAny <CancellationToken>()), Times.Exactly(retryWorkers)); }
public async Task WriteAsync_ShouldReturnSumOfWorkersResults_InsertionAndRetryWorkersAreStarted(int insertionWorkers, int retryWorkers) { // Arrange var result = new WorkerResult(3L, 1L); _workerFactoryMock.Setup(f => f.RunWorker(It.IsAny <ChannelWriter <ReplaceOneModel <BsonDocument> > >(), It.IsAny <ILogger>(), It.IsAny <CancellationToken>())) .ReturnsAsync(result); _workerFactoryMock.Setup(f => f.RunRetryWorker(It.IsAny <ChannelReader <ReplaceOneModel <BsonDocument> > >(), It.IsAny <ILogger>(), It.IsAny <CancellationToken>())) .ReturnsAsync(result); var writer = new DocumentsWriter(insertionWorkers, retryWorkers, _fixture.Create <string>(), _workerFactoryMock.Object, _loggerMock.Object); // Act var actual = await writer.WriteAsync(CancellationToken.None); // Assert var expected = new TransferResults(result.Successful * insertionWorkers, result.Successful * retryWorkers, result.Failed * (insertionWorkers + retryWorkers)); actual.Should().Be(expected); }
public CharBlockPool(DocumentsWriter docWriter) { InitBlock(); this.docWriter = docWriter; }
public IntBlockPool(DocumentsWriter docWriter, bool trackAllocations) { InitBlock(); this.docWriter = docWriter; this.trackAllocations = trackAllocations; }
int bufferUpto = -1; // Which buffer we are upto #endregion Fields #region Constructors public CharBlockPool(DocumentsWriter docWriter) { this.docWriter = docWriter; }
/// <summary> Initialize the deleter: find all previous commits in /// the Directory, incref the files they reference, call /// the policy to let it delete commits. This will remove /// any files not referenced by any of the commits. /// </summary> /// <throws> CorruptIndexException if the index is corrupt </throws> /// <throws> IOException if there is a low-level IO error </throws> public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.StreamWriter infoStream, DocumentsWriter docWriter, HashSet <string> synced) { this.docWriter = docWriter; this.infoStream = infoStream; this.synced = synced; if (infoStream != null) { Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy); } this.policy = policy; this.directory = directory; // First pass: walk the files and initialize our ref // counts: long currentGen = segmentInfos.Generation; IndexFileNameFilter filter = IndexFileNameFilter.Filter; System.String[] files = directory.ListAll(); CommitPoint currentCommitPoint = null; for (int i = 0; i < files.Length; i++) { System.String fileName = files[i]; if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN)) { // Add this file to refCounts with initial count 0: GetRefCount(fileName); if (fileName.StartsWith(IndexFileNames.SEGMENTS)) { // This is a commit (segments or segments_N), and // it's valid (<= the max gen). Load it, then // incref all files it refers to: if (infoStream != null) { Message("init: load commit \"" + fileName + "\""); } SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, fileName); } catch (System.IO.FileNotFoundException) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist if (infoStream != null) { Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point"); } sis = null; } catch (System.IO.IOException) { if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen) { throw; } else { // Most likely we are opening an index that // has an aborted "future" commit, so suppress // exc in this case sis = null; } } if (sis != null) { CommitPoint commitPoint = new CommitPoint(this, commitsToDelete, directory, sis); if (sis.Generation == segmentInfos.Generation) { currentCommitPoint = commitPoint; } commits.Add(commitPoint); IncRef(sis, true); if (lastSegmentInfos == null || sis.Generation > lastSegmentInfos.Generation) { lastSegmentInfos = sis; } } } } } if (currentCommitPoint == null) { // We did not in fact see the segments_N file // corresponding to the segmentInfos that was passed // in. Yet, it must exist, because our caller holds // the write lock. This can happen when the directory // listing was stale (eg when index accessed via NFS // client with stale directory listing cache). So we // try now to explicitly open this commit point: SegmentInfos sis = new SegmentInfos(); try { sis.Read(directory, segmentInfos.GetCurrentSegmentFileName()); } catch (System.IO.IOException) { throw new CorruptIndexException("failed to locate current segments_N file"); } if (infoStream != null) { Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName()); } currentCommitPoint = new CommitPoint(this, commitsToDelete, directory, sis); commits.Add(currentCommitPoint); IncRef(sis, true); } // We keep commits list in sorted order (oldest to newest): commits.Sort(); // Now delete anything with ref count at 0. These are // presumably abandoned files eg due to crash of // IndexWriter. foreach (KeyValuePair <string, RefCount> entry in refCounts) { string fileName = entry.Key; RefCount rc = refCounts[fileName]; if (0 == rc.count) { if (infoStream != null) { Message("init: removing unreferenced file \"" + fileName + "\""); } DeleteFile(fileName); } } // Finally, give policy a chance to remove things on // startup: policy.OnInit(commits); // Always protect the incoming segmentInfos since // sometime it may not be the most recent commit Checkpoint(segmentInfos, false); startingCommitDeleted = currentCommitPoint.IsDeleted; DeleteCommits(); }
public TermVectorsTermsWriter(DocumentsWriter docWriter) { InitBlock(); this.docWriter = docWriter; }
public CharBlockPool(DocumentsWriter docWriter) { this.docWriter = docWriter; }
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ namespace Lucene.Net.Index { internal abstract class DocConsumer { internal abstract DocConsumerPerThread addThread(DocumentsWriterThreadState perThread); internal abstract void Flush(System.Collections.Generic.ICollection<object> threads, DocumentsWriter.FlushState state); internal abstract void closeDocStore(DocumentsWriter.FlushState state); internal abstract void abort(); internal abstract bool freeRAM(); } }
public IntBlockPool(DocumentsWriter docWriter, bool trackAllocations) { this.docWriter = docWriter; this.trackAllocations = trackAllocations; }
public StoredFieldsWriter(DocumentsWriter docWriter, FieldInfos fieldInfos) { InitBlock(); this.docWriter = docWriter; this.fieldInfos = fieldInfos; }