public void TestSubclassConcurrentMergeScheduler()
{
MockDirectoryWrapper dir = NewMockDirectory();
dir.FailOn(new FailOnlyOnMerge());
Document doc = new Document();
Field idField = NewStringField("id", "", Field.Store.YES);
doc.Add(idField);
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergeScheduler(new MyMergeScheduler(this)).SetMaxBufferedDocs(2).SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetMergePolicy(NewLogMergePolicy()));
LogMergePolicy logMP = (LogMergePolicy)writer.Config.MergePolicy;
logMP.MergeFactor = 10;
for (int i = 0; i < 20; i++)
{
writer.AddDocument(doc);
}
((MyMergeScheduler)writer.Config.MergeScheduler).Sync();
writer.Dispose();
Assert.IsTrue(MergeThreadCreated);
Assert.IsTrue(MergeCalled);
Assert.IsTrue(ExcCalled);
dir.Dispose();
}
public void TestExceptionOnBackgroundThreadIsPropagatedToCallingThread()
{
using (MockDirectoryWrapper dir = NewMockDirectory())
{
dir.FailOn(new FailOnlyOnMerge());
Document doc = new Document();
Field idField = NewStringField("id", "", Field.Store.YES);
doc.Add(idField);
var mergeScheduler = new TaskMergeScheduler();
using (IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(mergeScheduler).SetMaxBufferedDocs(2).SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetMergePolicy(NewLogMergePolicy())))
{
LogMergePolicy logMP = (LogMergePolicy)writer.Config.MergePolicy;
logMP.MergeFactor = 10;
for (int i = 0; i < 20; i++)
{
writer.AddDocument(doc);
}
bool exceptionHit = false;
try
{
mergeScheduler.Sync();
}
catch (MergePolicy.MergeException)
{
exceptionHit = true;
}
assertTrue(exceptionHit);
}
}
}
protected override IndexWriterConfig CreateIndexWriterConfig(OpenMode openMode)
{
IndexWriterConfig conf = base.CreateIndexWriterConfig(openMode);
LogMergePolicy lmp = (LogMergePolicy)conf.MergePolicy;
lmp.MergeFactor = 2;
return(conf);
}
public override void BeforeClass()
{
base.BeforeClass();
// NOTE: turn off compound file, this test will open some index files directly.
OldFormatImpersonationIsActive = true;
IndexWriterConfig config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random, MockTokenizer.KEYWORD, false)).SetUseCompoundFile(false);
TermIndexInterval = config.TermIndexInterval;
IndexDivisor = TestUtil.NextInt32(Random, 1, 10);
NUMBER_OF_DOCUMENTS = AtLeast(100);
NUMBER_OF_FIELDS = AtLeast(Math.Max(10, 3 * TermIndexInterval * IndexDivisor / NUMBER_OF_DOCUMENTS));
Directory = NewDirectory();
config.SetCodec(new PreFlexRWCodec());
LogMergePolicy mp = NewLogMergePolicy();
// NOTE: turn off compound file, this test will open some index files directly.
mp.NoCFSRatio = 0.0;
config.SetMergePolicy(mp);
Populate(Directory, config);
DirectoryReader r0 = IndexReader.Open(Directory);
SegmentReader r = LuceneTestCase.GetOnlySegmentReader(r0);
string segment = r.SegmentName;
r.Dispose();
FieldInfosReader infosReader = (new PreFlexRWCodec()).FieldInfosFormat.FieldInfosReader;
FieldInfos fieldInfos = infosReader.Read(Directory, segment, "", IOContext.READ_ONCE);
string segmentFileName = IndexFileNames.SegmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION);
long tiiFileLength = Directory.FileLength(segmentFileName);
IndexInput input = Directory.OpenInput(segmentFileName, NewIOContext(Random));
TermEnum = new SegmentTermEnum(Directory.OpenInput(IndexFileNames.SegmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), NewIOContext(Random)), fieldInfos, false);
int totalIndexInterval = TermEnum.indexInterval * IndexDivisor;
SegmentTermEnum indexEnum = new SegmentTermEnum(input, fieldInfos, true);
Index = new TermInfosReaderIndex(indexEnum, IndexDivisor, tiiFileLength, totalIndexInterval);
indexEnum.Dispose();
input.Dispose();
Reader = IndexReader.Open(Directory);
SampleTerms = Sample(Random, Reader, 1000);
}
internal static MergePolicy NewSortingMergePolicy(Sort sort)
{
// create a MP with a low merge factor so that many merges happen
MergePolicy mp;
if (Random.nextBoolean())
{
TieredMergePolicy tmp = NewTieredMergePolicy(Random);
int numSegs = TestUtil.NextInt32(Random, 3, 5);
tmp.SegmentsPerTier = (numSegs);
tmp.MaxMergeAtOnce = (TestUtil.NextInt32(Random, 2, numSegs));
mp = tmp;
}
else
{
LogMergePolicy lmp = NewLogMergePolicy(Random);
lmp.MergeFactor = TestUtil.NextInt32(Random, 3, 5);
mp = lmp;
}
// wrap it with a sorting mp
return(new SortingMergePolicy(mp, sort));
}
public static IndexWriterConfig CreateWriterConfig(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit)
{
// :Post-Release-Update-Version.LUCENE_XY:
LuceneVersion version = (LuceneVersion)Enum.Parse(typeof(LuceneVersion), config.Get("writer.version", LuceneVersion.LUCENE_48.ToString()));
IndexWriterConfig iwConf = new IndexWriterConfig(version, runData.Analyzer);
iwConf.OpenMode = mode;
IndexDeletionPolicy indexDeletionPolicy = GetIndexDeletionPolicy(config);
iwConf.IndexDeletionPolicy = indexDeletionPolicy;
if (commit != null)
{
iwConf.IndexCommit = commit;
}
string mergeScheduler = config.Get("merge.scheduler",
"Lucene.Net.Index.ConcurrentMergeScheduler, Lucene.Net");
#if !FEATURE_CONCURRENTMERGESCHEDULER
// LUCENENET specific - hack to get our TaskMergeScheduler
// when a ConcurrentMergeScheduler is requested.
if (mergeScheduler.Contains(".ConcurrentMergeScheduler,"))
{
mergeScheduler = "Lucene.Net.Index.TaskMergeScheduler, Lucene.Net";
}
#endif
Type mergeSchedulerType = Type.GetType(mergeScheduler);
if (mergeSchedulerType == null)
{
throw new Exception("Unrecognized merge scheduler type '" + mergeScheduler + "'");
}
else if (mergeSchedulerType.Equals(typeof(NoMergeScheduler)))
{
iwConf.MergeScheduler = NoMergeScheduler.INSTANCE;
}
else
{
try
{
iwConf.MergeScheduler = (IMergeScheduler)Activator.CreateInstance(mergeSchedulerType);
}
catch (Exception e)
{
throw new Exception("unable to instantiate class '" + mergeScheduler + "' as merge scheduler", e);
}
if (mergeScheduler.Equals("Lucene.Net.Index.ConcurrentMergeScheduler", StringComparison.Ordinal))
{
#if FEATURE_CONCURRENTMERGESCHEDULER
ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler)iwConf.MergeScheduler;
int maxThreadCount = config.Get("concurrent.merge.scheduler.max.thread.count", ConcurrentMergeScheduler.DEFAULT_MAX_THREAD_COUNT);
int maxMergeCount = config.Get("concurrent.merge.scheduler.max.merge.count", ConcurrentMergeScheduler.DEFAULT_MAX_MERGE_COUNT);
#else
TaskMergeScheduler cms = (TaskMergeScheduler)iwConf.MergeScheduler;
int maxThreadCount = config.Get("concurrent.merge.scheduler.max.thread.count", 1);
int maxMergeCount = config.Get("concurrent.merge.scheduler.max.merge.count", 2);
#endif
cms.SetMaxMergesAndThreads(maxMergeCount, maxThreadCount);
}
}
string defaultCodec = config.Get("default.codec", null);
if (defaultCodec != null)
{
try
{
Type clazz = Type.GetType(defaultCodec);
iwConf.Codec = (Codec)Activator.CreateInstance(clazz);
}
catch (Exception e)
{
throw new Exception("Couldn't instantiate Codec: " + defaultCodec, e);
}
}
string mergePolicy = config.Get("merge.policy",
"Lucene.Net.Index.LogByteSizeMergePolicy, Lucene.Net");
bool isCompound = config.Get("compound", true);
Type mergePolicyType = Type.GetType(mergePolicy);
if (mergePolicyType == null)
{
throw new Exception("Unrecognized merge policy type '" + mergePolicy + "'");
}
else if (mergePolicyType.Equals(typeof(NoMergePolicy)))
{
iwConf.MergePolicy = isCompound ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES;
}
else
{
try
{
iwConf.MergePolicy = (MergePolicy)Activator.CreateInstance(mergePolicyType);
}
catch (Exception e)
{
throw new Exception("unable to instantiate class '" + mergePolicy + "' as merge policy", e);
}
iwConf.MergePolicy.NoCFSRatio = isCompound ? 1.0 : 0.0;
if (iwConf.MergePolicy is LogMergePolicy)
{
LogMergePolicy logMergePolicy = (LogMergePolicy)iwConf.MergePolicy;
logMergePolicy.MergeFactor = config.Get("merge.factor", OpenIndexTask.DEFAULT_MERGE_PFACTOR);
}
}
double ramBuffer = config.Get("ram.flush.mb", OpenIndexTask.DEFAULT_RAM_FLUSH_MB);
int maxBuffered = config.Get("max.buffered", OpenIndexTask.DEFAULT_MAX_BUFFERED);
if (maxBuffered == IndexWriterConfig.DISABLE_AUTO_FLUSH)
{
iwConf.RAMBufferSizeMB = ramBuffer;
iwConf.MaxBufferedDocs = maxBuffered;
}
else
{
iwConf.MaxBufferedDocs = maxBuffered;
iwConf.RAMBufferSizeMB = ramBuffer;
}
return(iwConf);
}
