public void before()
{
MemoryStream termIndexStream = new MemoryStream();
using (FileIndexWriter<string, IList<Posting>> indexWriter = new FileIndexWriter<string, IList<Posting>>(
new StringEncoder(), new PostingListEncoder(), termIndexStream))
{
this.postingsWithBar = new List<Posting>();
postingsWithBar.Add(new Posting(0, 1));
postingsWithBar.Add(new Posting(1, 2));
indexWriter.Add("bar", postingsWithBar);
this.postingsWithFoo = new List<Posting>();
postingsWithFoo.Add(new Posting(0, 4));
indexWriter.Add("foo", postingsWithFoo);
indexWriter.WriteOut();
}
this.index = new TermIndex(termIndexStream);
MemoryStream metadataStream = new MemoryStream();
using (CollectionMetadataWriter metadataWriter = new CollectionMetadataWriter(metadataStream))
{
metadataWriter.AddDocumentInfo(0, new DocumentInfo("http://www.example.com/index.html", "Example", 100, "", null));
metadataWriter.AddDocumentInfo(1, new DocumentInfo("http://www.example.com/menu.html", "Example", 300, "", null));
metadataWriter.WriteOut();
}
this.metadata = new IndexMetadata(metadataStream);
}
public SpimiIndexer(ILexer lexer)
{
this.lexer = lexer;
this.blockReader = new SpimiBlockReader();
this.blockWriter = new SpimiBlockWriter();
this.fileIndexWriter = new FileIndexWriter();
}
public CollectionMetadataWriter(Stream stream)
{
this.stream = stream;
// Start the index past the head
this.stream.Seek(HeaderSize, SeekOrigin.Begin);
this.documentsInfo = new FileIndexWriter<long, DocumentInfo>(
new LongEncoder(),
new DocumentInfoEncoder(), stream);
}
void PerformWrite(Stream stream)
{
FileIndexWriter writer = new FileIndexWriter();
List<PostingList> postingLists = new List<PostingList>();
List<string> docs = new List<string>();
docs.Add("aDoc");
docs.Add("bDoc");
postingLists.Add(new PostingList("aTerm", docs));
docs = new List<string>();
docs.Add("aDoc");
docs.Add("zDoc");
docs.Add("tDoc");
postingLists.Add(new PostingList("bTerm", docs));
writer.Write(stream, postingLists);
}
void PerformWrite(Stream stream)
{
long previousPosition = stream.Position;
FileIndexWriter<string, IList<Posting>> writer = new FileIndexWriter<string, IList<Posting>>(
new StringEncoder(),
new PostingListEncoder(),
stream);
List<Posting> postings = new List<Posting>();
postings.Add(new Posting(DocA, 1));
postings.Add(new Posting(DocB, 1));
writer.Add("aTerm", postings);
postings = new List<Posting>();
postings.Add(new Posting(DocA, 1));
postings.Add(new Posting(DocZ, 1));
postings.Add(new Posting(DocT, 1));
writer.Add("bTerm", postings);
string file = Path.GetTempFileName();
writer.WriteOut();
Assert.AreEqual(previousPosition, stream.Position);
}
private void MergeBlocks()
{
if (termIndexBlockWriter.Postings > 0)
FlushBlockWriter();
using (FileIndexWriter<string, IList<Posting>> writer = new FileIndexWriter<string, IList<Posting>>(
new StringEncoder(),
new PostingListEncoder(), indexStream))
{
SpimiBlockReader blockReader = new SpimiBlockReader();
List<IEnumerator<PostingList>> openedBlocks = blockReader.OpenBlocks(this.termIndexBlockFilePaths);
foreach (PostingList postingList in blockReader.BeginBlockMerge(openedBlocks))
{
writer.Add(postingList.Term, postingList.Postings);
}
writer.WriteOut();
}
}
