protected override MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge)
{
MergeThread thread = new MyMergeThread(this, writer, merge);
thread.ThreadPriority = MergeThreadPriority;
thread.SetDaemon(true);
thread.Name = "MyMergeThread";
return thread;
}
private void DoTestSearch(Random random, StringWriter @out, bool useCompoundFile)
{
Store.Directory directory = NewDirectory();
Analyzer analyzer = new MockAnalyzer(random);
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
MergePolicy mp = conf.MergePolicy;
mp.NoCFSRatio = useCompoundFile ? 1.0 : 0.0;
IndexWriter writer = new IndexWriter(directory, conf);
string[] docs = new string[] { "a b c d e", "a b c d e a b c d e", "a b c d e f g h i j", "a c e", "e c a", "a c e a c e", "a c e a b c" };
for (int j = 0; j < docs.Length; j++)
{
Documents.Document d = new Documents.Document();
d.Add(NewTextField("contents", docs[j], Field.Store.YES));
d.Add(NewStringField("id", "" + j, Field.Store.NO));
writer.AddDocument(d);
}
writer.Dispose();
IndexReader reader = DirectoryReader.Open(directory);
IndexSearcher searcher = NewSearcher(reader);
ScoreDoc[] hits = null;
Sort sort = new Sort(SortField.FIELD_SCORE, new SortField("id", SortFieldType.INT32));
foreach (Query query in BuildQueries())
{
@out.WriteLine("Query: " + query.ToString("contents"));
if (Verbose)
{
Console.WriteLine("TEST: query=" + query);
}
hits = searcher.Search(query, null, 1000, sort).ScoreDocs;
@out.WriteLine(hits.Length + " total results");
for (int i = 0; i < hits.Length && i < 10; i++)
{
Documents.Document d = searcher.Doc(hits[i].Doc);
@out.WriteLine(i + " " + hits[i].Score + " " + d.Get("contents"));
}
}
reader.Dispose();
directory.Dispose();
}
internal int[] starts; // used for binary search of mapped docID
#endregion Fields
#region Constructors
public MergeDocIDRemapper(SegmentInfos infos, int[][] docMaps, int[] delCounts, MergePolicy.OneMerge merge, int mergedDocCount)
{
this.docMaps = docMaps;
SegmentInfo firstSegment = merge.segments.Info(0);
int i = 0;
while (true)
{
SegmentInfo info = infos.Info(i);
if (info.Equals(firstSegment))
break;
minDocID += info.docCount;
i++;
}
int numDocs = 0;
for (int j = 0; j < docMaps.Length; i++, j++)
{
numDocs += infos.Info(i).docCount;
System.Diagnostics.Debug.Assert(infos.Info(i).Equals(merge.segments.Info(j)));
}
maxDocID = minDocID + numDocs;
starts = new int[docMaps.Length];
newStarts = new int[docMaps.Length];
starts[0] = minDocID;
newStarts[0] = minDocID;
for (i = 1; i < docMaps.Length; i++)
{
int lastDocCount = merge.segments.Info(i - 1).docCount;
starts[i] = starts[i - 1] + lastDocCount;
newStarts[i] = newStarts[i - 1] + lastDocCount - delCounts[i - 1];
}
docShift = numDocs - mergedDocCount;
// There are rare cases when docShift is 0. It happens
// if you try to delete a docID that's out of bounds,
// because the SegmentReader still allocates deletedDocs
// and pretends it has deletions ... so we can't make
// this assert here
// assert docShift > 0;
// Make sure it all adds up:
System.Diagnostics.Debug.Assert(docShift == maxDocID - (newStarts[docMaps.Length - 1] + merge.segments.Info(docMaps.Length - 1).docCount - delCounts[docMaps.Length - 1]));
}
/// <summary>
/// Just tries to configure things to keep the open file
/// count lowish.
/// </summary>
public static void ReduceOpenFiles(IndexWriter w)
{
// keep number of open files lowish
MergePolicy mp = w.Config.MergePolicy;
if (mp is LogMergePolicy lmp)
{
lmp.MergeFactor = Math.Min(5, lmp.MergeFactor);
lmp.NoCFSRatio = 1.0;
}
else if (mp is TieredMergePolicy tmp)
{
tmp.MaxMergeAtOnce = Math.Min(5, tmp.MaxMergeAtOnce);
tmp.SegmentsPerTier = Math.Min(5, tmp.SegmentsPerTier);
tmp.NoCFSRatio = 1.0;
}
IMergeScheduler ms = w.Config.MergeScheduler;
if (ms is IConcurrentMergeScheduler concurrentMergeScheduler)
{
// wtf... shouldnt it be even lower since its 1 by default?!?!
concurrentMergeScheduler.SetMaxMergesAndThreads(3, 2);
}
}
protected override void DoMerge(MergePolicy.OneMerge merge)
{
OuterInstance.MergeCalled = true;
base.DoMerge(merge);
}
public MyMergeThread(TestMergeSchedulerExternal.MyMergeScheduler outerInstance, IndexWriter writer, MergePolicy.OneMerge merge)
: base(outerInstance, writer, merge)
{
this.OuterInstance = outerInstance;
outerInstance.OuterInstance.MergeThreadCreated = true;
}
/// <summary>
/// Builder method for <see cref="IndexWriterConfig.MergePolicy"/>.
/// </summary>
/// <param name="config">this <see cref="IndexWriterConfig"/> instance</param>
/// <param name="mergePolicy"></param>
/// <returns>this <see cref="IndexWriterConfig"/> instance</returns>
public static IndexWriterConfig SetMergePolicy(this IndexWriterConfig config, MergePolicy mergePolicy)
{
config.MergePolicy = mergePolicy;
return(config);
}
private bool IsMergePolicyEnabled(MergePolicy mergePolicy)
{
int tmpMergePolicy = (int)mergePolicy;
return(((int)this.MergePolicy & tmpMergePolicy) == tmpMergePolicy);
}
public MergePolicyAttribute(MergePolicy policy)
{
this.Policy = policy;
}
public SingleResult(bool?success, string message, MergePolicy policy)
{
Success = success;
Message = message;
Policy = policy;
}
/// <summary>
/// Create a new <see cref="MergePolicy"/> that sorts documents with the given <paramref name="sort"/>.
/// </summary>
public SortingMergePolicy(MergePolicy @in, Sort sort)
{
this.@in = @in;
this.sorter = new Sorter(sort);
this.sort = sort;
}
/// <summary>
/// Create a new {@code MergePolicy} that sorts documents with the given {@code sort}. </summary>
public SortingMergePolicy(MergePolicy @in, Sort sort)
{
this.@in = @in;
this.sorter = new Sorter(sort);
this.sort = sort;
}
public UpdateSubscriptionPopUp(string path,
ILogger logger,
Subscription subscription,
IEnumerable <string> suggestedChannels,
IEnumerable <string> suggestedRepositories,
IEnumerable <string> availableUpdateFrequencies,
IEnumerable <string> availableMergePolicyHelp)
: base(path, logger)
{
_logger = logger;
List <MergePolicy> mergePolicies = new List <MergePolicy>();
// This is a workaround issue https://github.com/aaubry/YamlDotNet/issues/383 which
// is causing to display empty items in the properties collection of the mergePolicies
try
{
foreach (MergePolicy mergePolicy in subscription.Policy.MergePolicies)
{
MergePolicy policy = new MergePolicy
{
Name = mergePolicy.Name,
Properties = ImmutableDictionary.Create <string, JToken>(),
};
foreach (string key in mergePolicy.Properties.Keys)
{
JToken value = JsonConvert.DeserializeObject <JToken>(
mergePolicy.Properties[key].ToString());
policy.Properties = policy.Properties.Add(key, value);
}
mergePolicies.Add(policy);
}
}
catch
{
// Something failed parsing the properties. Continue with the original collection
mergePolicies = new List <MergePolicy>(subscription.Policy.MergePolicies);
}
_yamlData = new SubscriptionData
{
Id = GetCurrentSettingForDisplay(subscription.Id.ToString(), subscription.Id.ToString(), false),
Channel = GetCurrentSettingForDisplay(subscription.Channel.Name, subscription.Channel.Name, false),
SourceRepository = GetCurrentSettingForDisplay(subscription.SourceRepository, subscription.SourceRepository, false),
Batchable = GetCurrentSettingForDisplay(subscription.Policy.Batchable.ToString(), subscription.Policy.Batchable.ToString(), false),
UpdateFrequency = GetCurrentSettingForDisplay(subscription.Policy.UpdateFrequency.ToString(), subscription.Policy.UpdateFrequency.ToString(), false),
Enabled = GetCurrentSettingForDisplay(subscription.Enabled.ToString(), subscription.Enabled.ToString(), false),
MergePolicies = mergePolicies
};
ISerializer serializer = new SerializerBuilder().Build();
string yaml = serializer.Serialize(_yamlData);
string[] lines = yaml.Split(Environment.NewLine);
// Initialize line contents. Augment the input lines with suggestions and explanation
Contents = new Collection <Line>(new List <Line>
{
new Line($"Use this form to update the values of subscription '{subscription.Id}'.", true),
new Line($"Note that if you are setting 'Is batchable' to true you need to remove all Merge Policies.", true),
new Line()
});
foreach (string line in lines)
{
Contents.Add(new Line(line));
}
// Add helper comments
Contents.Add(new Line($"Suggested repository URLs for '{SubscriptionData.sourceRepoElement}':", true));
foreach (string suggestedRepo in suggestedRepositories)
{
Contents.Add(new Line($" {suggestedRepo}", true));
}
Contents.Add(new Line("", true));
Contents.Add(new Line("Suggested Channels", true));
foreach (string suggestedChannel in suggestedChannels)
{
Contents.Add(new Line($" {suggestedChannel}", true));
}
Contents.Add(new Line("", true));
Contents.Add(new Line("Available Merge Policies", true));
foreach (string mergeHelp in availableMergePolicyHelp)
{
Contents.Add(new Line($" {mergeHelp}", true));
}
}