public DocumentsWriterThreadState(DocumentsWriter docWriter)
{
this.docWriter = docWriter;
docState = new DocumentsWriter.DocState();
docState.maxFieldLength = docWriter.maxFieldLength;
docState.infoStream = docWriter.infoStream;
docState.similarity = docWriter.similarity;
docState.docWriter = docWriter;
consumer = docWriter.consumer.AddThread(this);
}
public DocumentsWriterThreadState(DocumentsWriter docWriter)
{
this.docWriter = docWriter;
docState = new DocumentsWriter.DocState();
docState.maxFieldLength = docWriter.maxFieldLength;
docState.infoStream = docWriter.infoStream;
docState.similarity = docWriter.similarity;
docState.docWriter = docWriter;
docState.allowMinus1Position = docWriter.writer.GetAllowMinus1Position();
consumer = docWriter.consumer.AddThread(this);
}
public SegmentWriteState(DocumentsWriter docWriter, Directory directory, System.String segmentName, System.String docStoreSegmentName, int numDocs, int numDocsInStore, int termIndexInterval)
{
this.docWriter = docWriter;
this.directory = directory;
this.segmentName = segmentName;
this.docStoreSegmentName = docStoreSegmentName;
this.numDocs = numDocs;
this.numDocsInStore = numDocsInStore;
this.termIndexInterval = termIndexInterval;
flushedFiles = new System.Collections.Generic.HashSet <string>();
}
public TermsHash(DocumentsWriter docWriter, bool trackAllocations, TermsHashConsumer consumer, TermsHash nextTermsHash)
{
this.docWriter = docWriter;
this.consumer = consumer;
this.nextTermsHash = nextTermsHash;
this.trackAllocations = trackAllocations;
// Why + 4*POINTER_NUM_BYTE below?
// +1: Posting is referenced by postingsFreeList array
// +3: Posting is referenced by hash, which
// targets 25-50% fill factor; approximate this
// as 3X # pointers
bytesPerPosting = consumer.BytesPerPosting() + 4 * DocumentsWriter.POINTER_NUM_BYTE;
postingsFreeChunk = (int)(DocumentsWriter.BYTE_BLOCK_SIZE / bytesPerPosting);
}
public async Task WriteAsync_ShouldPassNullToRetryChannelForInsertionWorkers_RetryWorkersCountIsZero()
{
// Arrange
_workerFactoryMock.Setup(f => f.RunWorker(It.IsAny <ChannelWriter <ReplaceOneModel <BsonDocument> > >(),
It.IsAny <ILogger>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(_fixture.Create <WorkerResult>());
_workerFactoryMock.Setup(f => f.RunRetryWorker(It.IsAny <ChannelReader <ReplaceOneModel <BsonDocument> > >(),
It.IsAny <ILogger>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(_fixture.Create <WorkerResult>());
var writer = new DocumentsWriter(10, 0, _fixture.Create <string>(),
_workerFactoryMock.Object, _loggerMock.Object);
// Act
await writer.WriteAsync(CancellationToken.None);
// Assert
_workerFactoryMock.Verify(f => f.RunWorker(null, It.IsAny <ILogger>(), It.IsAny <CancellationToken>()),
Times.Exactly(10));
}
public async Task WriteAsync_ShouldStartExactNumberOfWorkers_NumberOfInsertionWorkersGreaterThanZero(int insertionWorkers, int retryWorkers)
{
// Arrange
_workerFactoryMock.Setup(f => f.RunWorker(It.IsAny <ChannelWriter <ReplaceOneModel <BsonDocument> > >(),
It.IsAny <ILogger>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(new WorkerResult(100, 0));
_workerFactoryMock.Setup(f => f.RunRetryWorker(It.IsAny <ChannelReader <ReplaceOneModel <BsonDocument> > >(),
It.IsAny <ILogger>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(new WorkerResult(100, 0));
var writer = new DocumentsWriter(insertionWorkers, retryWorkers, _fixture.Create <string>(),
_workerFactoryMock.Object, _loggerMock.Object);
// Act
await writer.WriteAsync(CancellationToken.None);
// Assert
_workerFactoryMock.Verify(f => f.RunWorker(It.IsAny <ChannelWriter <ReplaceOneModel <BsonDocument> > >(),
It.IsAny <ILogger>(), It.IsAny <CancellationToken>()), Times.Exactly(insertionWorkers));
_workerFactoryMock.Verify(f => f.RunRetryWorker(It.IsAny <ChannelReader <ReplaceOneModel <BsonDocument> > >(),
It.IsAny <ILogger>(), It.IsAny <CancellationToken>()), Times.Exactly(retryWorkers));
}
public async Task WriteAsync_ShouldReturnSumOfWorkersResults_InsertionAndRetryWorkersAreStarted(int insertionWorkers, int retryWorkers)
{
// Arrange
var result = new WorkerResult(3L, 1L);
_workerFactoryMock.Setup(f => f.RunWorker(It.IsAny <ChannelWriter <ReplaceOneModel <BsonDocument> > >(),
It.IsAny <ILogger>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(result);
_workerFactoryMock.Setup(f => f.RunRetryWorker(It.IsAny <ChannelReader <ReplaceOneModel <BsonDocument> > >(),
It.IsAny <ILogger>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(result);
var writer = new DocumentsWriter(insertionWorkers, retryWorkers, _fixture.Create <string>(),
_workerFactoryMock.Object, _loggerMock.Object);
// Act
var actual = await writer.WriteAsync(CancellationToken.None);
// Assert
var expected = new TransferResults(result.Successful * insertionWorkers, result.Successful * retryWorkers,
result.Failed * (insertionWorkers + retryWorkers));
actual.Should().Be(expected);
}
public CharBlockPool(DocumentsWriter docWriter)
{
InitBlock();
this.docWriter = docWriter;
}
public IntBlockPool(DocumentsWriter docWriter, bool trackAllocations)
{
InitBlock();
this.docWriter = docWriter;
this.trackAllocations = trackAllocations;
}
int bufferUpto = -1; // Which buffer we are upto
#endregion Fields
#region Constructors
public CharBlockPool(DocumentsWriter docWriter)
{
this.docWriter = docWriter;
}
/// <summary> Initialize the deleter: find all previous commits in
/// the Directory, incref the files they reference, call
/// the policy to let it delete commits. This will remove
/// any files not referenced by any of the commits.
/// </summary>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.StreamWriter infoStream, DocumentsWriter docWriter, HashSet <string> synced)
{
this.docWriter = docWriter;
this.infoStream = infoStream;
this.synced = synced;
if (infoStream != null)
{
Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy);
}
this.policy = policy;
this.directory = directory;
// First pass: walk the files and initialize our ref
// counts:
long currentGen = segmentInfos.Generation;
IndexFileNameFilter filter = IndexFileNameFilter.Filter;
System.String[] files = directory.ListAll();
CommitPoint currentCommitPoint = null;
for (int i = 0; i < files.Length; i++)
{
System.String fileName = files[i];
if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
{
// Add this file to refCounts with initial count 0:
GetRefCount(fileName);
if (fileName.StartsWith(IndexFileNames.SEGMENTS))
{
// This is a commit (segments or segments_N), and
// it's valid (<= the max gen). Load it, then
// incref all files it refers to:
if (infoStream != null)
{
Message("init: load commit \"" + fileName + "\"");
}
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, fileName);
}
catch (System.IO.FileNotFoundException)
{
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
if (infoStream != null)
{
Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
}
sis = null;
}
catch (System.IO.IOException)
{
if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen)
{
throw;
}
else
{
// Most likely we are opening an index that
// has an aborted "future" commit, so suppress
// exc in this case
sis = null;
}
}
if (sis != null)
{
CommitPoint commitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
if (sis.Generation == segmentInfos.Generation)
{
currentCommitPoint = commitPoint;
}
commits.Add(commitPoint);
IncRef(sis, true);
if (lastSegmentInfos == null || sis.Generation > lastSegmentInfos.Generation)
{
lastSegmentInfos = sis;
}
}
}
}
}
if (currentCommitPoint == null)
{
// We did not in fact see the segments_N file
// corresponding to the segmentInfos that was passed
// in. Yet, it must exist, because our caller holds
// the write lock. This can happen when the directory
// listing was stale (eg when index accessed via NFS
// client with stale directory listing cache). So we
// try now to explicitly open this commit point:
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, segmentInfos.GetCurrentSegmentFileName());
}
catch (System.IO.IOException)
{
throw new CorruptIndexException("failed to locate current segments_N file");
}
if (infoStream != null)
{
Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName());
}
currentCommitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
commits.Add(currentCommitPoint);
IncRef(sis, true);
}
// We keep commits list in sorted order (oldest to newest):
commits.Sort();
// Now delete anything with ref count at 0. These are
// presumably abandoned files eg due to crash of
// IndexWriter.
foreach (KeyValuePair <string, RefCount> entry in refCounts)
{
string fileName = entry.Key;
RefCount rc = refCounts[fileName];
if (0 == rc.count)
{
if (infoStream != null)
{
Message("init: removing unreferenced file \"" + fileName + "\"");
}
DeleteFile(fileName);
}
}
// Finally, give policy a chance to remove things on
// startup:
policy.OnInit(commits);
// Always protect the incoming segmentInfos since
// sometime it may not be the most recent commit
Checkpoint(segmentInfos, false);
startingCommitDeleted = currentCommitPoint.IsDeleted;
DeleteCommits();
}
public TermVectorsTermsWriter(DocumentsWriter docWriter)
{
InitBlock();
this.docWriter = docWriter;
}
public CharBlockPool(DocumentsWriter docWriter)
{
this.docWriter = docWriter;
}
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
namespace Lucene.Net.Index
{
internal abstract class DocConsumer
{
internal abstract DocConsumerPerThread addThread(DocumentsWriterThreadState perThread);
internal abstract void Flush(System.Collections.Generic.ICollection<object> threads, DocumentsWriter.FlushState state);
internal abstract void closeDocStore(DocumentsWriter.FlushState state);
internal abstract void abort();
internal abstract bool freeRAM();
}
}
public IntBlockPool(DocumentsWriter docWriter, bool trackAllocations)
{
this.docWriter = docWriter;
this.trackAllocations = trackAllocations;
}
public StoredFieldsWriter(DocumentsWriter docWriter, FieldInfos fieldInfos)
{
InitBlock();
this.docWriter = docWriter;
this.fieldInfos = fieldInfos;
}
