public void ExternalIndexBuildTest() { var stream = new MemoryStream(); var postingsListBuilder = new MixedPostingsListBuilder <string>(RangeThreshold); var buildableIndex = new DictonaryBasedExternalBuildableIndex <string>(postingsListBuilder, stream); var docs = new (DocumentId Id, string[] Text)[]
static void BuildExternalIndex() { PrepareOutputDirectory(externalIndexPath); var reader = new CorpusZipReader <IList <char> >(wikiPath, charDataSerializer); using var buildableIndex = new BlockedExternalBuildableIndex <int>( DictonaryBasedExternalBuildableIndex <int> .GetCreateMethodWithVarintPostingsLists(), externalIndexPath, BlockSize); var indexBuilder = new IndexBuilder <int, IEnumerable <int> >(buildableIndex); var processor = new WikitextProcessor(); indexBuilder.IndexCorpus(processor.Transform(reader.Read())); using var index = buildableIndex.Build(); var serializer = new ExternalIndexSerializer <int>(); serializer.Serialize(externalIndexPath, index); }
public void BuildBlockedExternalIndexTest_DictionaryBased_VarintPostingsList() { BuildBlockedExternalIndexTest(DictonaryBasedExternalBuildableIndex <string> .GetCreateMethodWithVarintPostingsLists()); }