public void Index(string field, TextReader indexedValue) { if (field == null) { throw new ArgumentNullException("field"); } if (field.Length > 256) { throw new ArgumentException("field name cannot exceed 256 characters", "field"); } if (indexedValue == null) { throw new ArgumentNullException("indexedValue"); } var treeName = "@fld_" + field; _source = _analyzer.CreateTokenSource(field, _source); _source.SetReader(indexedValue); while (_source.Next()) { if (_analyzer.Process(field, _source) == false) { continue; } var byteCount = Encoding.UTF8.GetByteCount(_source.Buffer, 0, _source.Size); if (byteCount > 256) { throw new IOException("Cannot index a term that is greater than 256 bytes, but got a term with " + byteCount + " bytes"); } var bytes = _bufferPool.Take(byteCount); _toBeFreed.Add(bytes); Encoding.UTF8.GetBytes(_source.Buffer, 0, _source.Size, bytes, 0); _writeBatch.MultiAdd(new Slice(bytes, (ushort)byteCount), _currentDocumentIdSlice, treeName); } }
public void AddField(string field, TextReader indexedValue = null, string storedValue = null, FieldOptions options = FieldOptions.None) { if (storedValue != null) { var num = _parent.GetFieldNumber(field); _binaryWriter.Write(num); _binaryWriter.Write(storedValue); _currentStoredCount++; } if (indexedValue == null) { if (storedValue == null) { throw new ArgumentException("It isn't meaningful to pass null for both indexedValue and storedValue"); } return; } _source = _analyzer.CreateTokenSource(field, _source); _source.SetReader(indexedValue); while (_source.Next()) { if (options.HasFlag(FieldOptions.NoAnalyzer) == false) { if (_analyzer.Process(field, _source) == false) { continue; } } var byteCount = Encoding.UTF8.GetByteCount(_source.Buffer, 0, _source.Size); var bytes = _bufferPool.Take(byteCount); Encoding.UTF8.GetBytes(_source.Buffer, 0, _source.Size, bytes, 0); Debug.Assert(byteCount < ushort.MaxValue); _currentTermCount++; var key = Tuple.Create(field, new ArraySegmentKey <byte>(bytes, byteCount)); TermInfo info; if (_currentTerms.TryGetValue(key, out info)) { _bufferPool.Return(bytes); } else { _currentTerms[key] = info = new TermInfo { Boost = 1.0f }; _usedBuffers.Add(bytes); } info.Freq++; if (options.HasFlag(FieldOptions.TermPositions)) { if (info.Positions == null) { info.Positions = new List <int>(); } info.Positions.Add(_source.Position); } } }
public void AddField(string field, TextReader indexedValue = null, string storedValue = null, FieldOptions options = FieldOptions.None) { if (storedValue != null) { var num = _parent.GetFieldNumber(field); _binaryWriter.Write(num); _binaryWriter.Write(storedValue); _currentStoredCount++; } if (indexedValue == null) { if (storedValue == null) throw new ArgumentException("It isn't meaningful to pass null for both indexedValue and storedValue"); return; } _source = _analyzer.CreateTokenSource(field, _source); _source.SetReader(indexedValue); while (_source.Next()) { if (options.HasFlag(FieldOptions.NoAnalyzer) == false) { if (_analyzer.Process(field, _source) == false) continue; } var byteCount = Encoding.UTF8.GetByteCount(_source.Buffer, 0, _source.Size); var bytes = _bufferPool.Take(byteCount); Encoding.UTF8.GetBytes(_source.Buffer, 0, _source.Size, bytes, 0); Debug.Assert(byteCount < ushort.MaxValue); _currentTermCount++; var key = Tuple.Create(field, new ArraySegmentKey<byte>(bytes, byteCount)); TermInfo info; if (_currentTerms.TryGetValue(key, out info)) { _bufferPool.Return(bytes); } else { _currentTerms[key] = info = new TermInfo {Boost = 1.0f}; _usedBuffers.Add(bytes); } info.Freq++; if (options.HasFlag(FieldOptions.TermPositions)) { if (info.Positions == null) info.Positions = new List<int>(); info.Positions.Add(_source.Position); } } }