public /*internal*/ FieldsReader(Directory d, System.String segment, FieldInfos fn) { fieldInfos = fn; fieldsStream = d.OpenFile(segment + ".fdt"); indexStream = d.OpenFile(segment + ".fdx"); size = (int) (indexStream.Length() / 8); }
private void Initialize(Directory directory, System.String segment, FieldInfos fis, bool isi) { fieldInfos = fis; isIndex = isi; output = directory.CreateFile(segment + (isIndex?".tii":".tis")); output.WriteInt(FORMAT); // write format output.WriteLong(0); // leave space for size output.WriteInt(indexInterval); // write indexInterval output.WriteInt(skipInterval); // write skipInterval }
public /*internal*/ TermInfosReader(Directory dir, System.String seg, FieldInfos fis) { directory = dir; segment = seg; fieldInfos = fis; origEnum = new SegmentTermEnum(directory.OpenFile(segment + ".tis"), fieldInfos, false); size = origEnum.size; ReadIndex(); }
/*internal*/ public void AddDocument(System.String segment, Document doc) { // write Field names fieldInfos = new FieldInfos(); fieldInfos.Add(doc); fieldInfos.Write(directory, segment + ".fnm"); // write Field values FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos); try { fieldsWriter.AddDocument(doc); } finally { fieldsWriter.Close(); } // invert doc into postingTable postingTable.Clear(); // clear postingTable fieldLengths = new int[fieldInfos.Size()]; // init fieldLengths fieldPositions = new int[fieldInfos.Size()]; // init fieldPositions fieldBoosts = new float[fieldInfos.Size()]; // init fieldBoosts float boost = doc.GetBoost(); for (int i = 0; i < fieldBoosts.Length; i++) { fieldBoosts[i] = boost; } InvertDocument(doc); // sort postingTable into an array Posting[] postings = SortPostingTable(); /* for (int i = 0; i < postings.length; i++) { Posting posting = postings[i]; System.out.print(posting.term); System.out.print(" freq=" + posting.freq); System.out.print(" pos="); System.out.print(posting.positions[0]); for (int j = 1; j < posting.freq; j++) System.out.print("," + posting.positions[j]); System.out.println(""); } */ // write postings WritePostings(postings, segment); // write norms of indexed fields WriteNorms(doc, segment); }
/// <summary>Create term vectors writer for the specified segment in specified /// directory. A new TermVectorsWriter should be created for each /// segment. The parameter <code>maxFields</code> indicates how many total /// fields are found in this document. Not all of these fields may require /// termvectors to be stored, so the number of calls to /// <code>openField</code> is less or equal to this number. /// </summary> public TermVectorsWriter(Directory directory, System.String segment, FieldInfos fieldInfos) { // Open files for TermVector storage tvx = directory.CreateFile(segment + TVX_EXTENSION); tvx.WriteInt(FORMAT_VERSION); tvd = directory.CreateFile(segment + TVD_EXTENSION); tvd.WriteInt(FORMAT_VERSION); tvf = directory.CreateFile(segment + TVF_EXTENSION); tvf.WriteInt(FORMAT_VERSION); this.fieldInfos = fieldInfos; fields = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(fieldInfos.Size())); terms = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(10)); }
public /*internal*/ TermVectorsReader(Directory d, System.String segment, FieldInfos fieldInfos) { if (d.FileExists(segment + TermVectorsWriter.TVX_EXTENSION)) { tvx = d.OpenFile(segment + TermVectorsWriter.TVX_EXTENSION); CheckValidFormat(tvx); tvd = d.OpenFile(segment + TermVectorsWriter.TVD_EXTENSION); CheckValidFormat(tvd); tvf = d.OpenFile(segment + TermVectorsWriter.TVF_EXTENSION); CheckValidFormat(tvf); size = (int) tvx.Length() / 8; } this.fieldInfos = fieldInfos; }
internal SegmentTermEnum(InputStream i, FieldInfos fis, bool isi) { input = i; fieldInfos = fis; isIndex = isi; int firstInt = input.ReadInt(); if (firstInt >= 0) { // original-format file, without explicit format version number format = 0; size = firstInt; // back-compatible settings indexInterval = 128; skipInterval = System.Int32.MaxValue; // switch off skipTo optimization } else { // we have a format version number format = firstInt; // check that it is a format we can understand if (format < TermInfosWriter.FORMAT) throw new System.IO.IOException("Unknown format version:" + format); size = input.ReadLong(); // read the size if (format == - 1) { if (!isIndex) { indexInterval = input.ReadInt(); formatM1SkipInterval = input.ReadInt(); } // switch off skipTo optimization for file format prior to 1.4rc2 in order to avoid a bug in // skipTo implementation of these versions skipInterval = System.Int32.MaxValue; } else { indexInterval = input.ReadInt(); skipInterval = input.ReadInt(); } } }
private void Initialize(SegmentInfo si) { segment = si.name; // Use compound file directory for some files, if it exists Directory cfsDir = Directory(); if (Directory().FileExists(segment + ".cfs")) { cfsReader = new CompoundFileReader(Directory(), segment + ".cfs"); cfsDir = cfsReader; } // No compound file exists - use the multi-file format fieldInfos = new FieldInfos(cfsDir, segment + ".fnm"); fieldsReader = new FieldsReader(cfsDir, segment, fieldInfos); tis = new TermInfosReader(cfsDir, segment, fieldInfos); // NOTE: the bitvector is stored using the regular directory, not cfs if (HasDeletions(si)) deletedDocs = new BitVector(Directory(), segment + ".del"); // make sure that all index files have been read or are kept open // so that if an index update removes them we'll still have them freqStream = cfsDir.OpenFile(segment + ".frq"); proxStream = cfsDir.OpenFile(segment + ".prx"); OpenNorms(cfsDir); if (fieldInfos.HasVectors()) { // open term vector files only as needed termVectorsReader = new TermVectorsReader(cfsDir, segment, fieldInfos); } }
private TermInfosWriter(Directory directory, System.String segment, FieldInfos fis, bool isIndex) { Initialize(directory, segment, fis, isIndex); }
public /*internal*/ TermInfosWriter(Directory directory, System.String segment, FieldInfos fis) { Initialize(directory, segment, fis, false); other = new TermInfosWriter(directory, segment, fis, true); other.other = this; }
/// <summary> </summary> /// <returns> The number of documents in all of the readers /// </returns> /// <throws> IOException </throws> private int MergeFields() { fieldInfos = new FieldInfos(); // merge Field names int docCount = 0; for (int i = 0; i < readers.Count; i++) { Monodoc.Lucene.Net.Index.IndexReader reader = (Monodoc.Lucene.Net.Index.IndexReader) readers[i]; fieldInfos.AddIndexed(reader.GetIndexedFieldNames(true), true); fieldInfos.AddIndexed(reader.GetIndexedFieldNames(false), false); fieldInfos.Add(reader.GetFieldNames(false), false); } fieldInfos.Write(directory, segment + ".fnm"); FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos); try { for (int i = 0; i < readers.Count; i++) { Monodoc.Lucene.Net.Index.IndexReader reader = (Monodoc.Lucene.Net.Index.IndexReader) readers[i]; int maxDoc = reader.MaxDoc(); for (int j = 0; j < maxDoc; j++) if (!reader.IsDeleted(j)) { // skip deleted docs fieldsWriter.AddDocument(reader.Document(j)); docCount++; } } } finally { fieldsWriter.Close(); } return docCount; }
internal FieldsWriter(Directory d, System.String segment, FieldInfos fn) { fieldInfos = fn; fieldsStream = d.CreateFile(segment + ".fdt"); indexStream = d.CreateFile(segment + ".fdx"); }