private TokenWrapper InitTokenWrapper(AttributeSource input) { if (onlyUseNewAPI) { // no wrapper needed return(null); } else { // if possible get the wrapper from the filter's input stream if (input is TokenStream && ((TokenStream)input).tokenWrapper != null) { return(((TokenStream)input).tokenWrapper); } // check that all attributes are implemented by the same TokenWrapper instance Attribute att = AddAttribute(typeof(TermAttribute)); if (att is TokenWrapper && AddAttribute(typeof(TypeAttribute)) == att && AddAttribute(typeof(PositionIncrementAttribute)) == att && AddAttribute(typeof(FlagsAttribute)) == att && AddAttribute(typeof(OffsetAttribute)) == att && AddAttribute(typeof(PayloadAttribute)) == att) { return((TokenWrapper)att); } else { throw new System.NotSupportedException("If onlyUseNewAPI is disabled, all basic Attributes must be implemented by the internal class " + "TokenWrapper. Please make sure, that all TokenStreams/TokenFilters in this chain have been " + "instantiated with this flag disabled and do not add any custom instances for the basic Attributes!"); } } }
/// <summary> Expert: Creates a token stream for numeric values with the specified /// <code>precisionStep</code> using the given {@link AttributeSource}. /// The stream is not yet initialized, /// before using set a value using the various set<em>???</em>Value() methods. /// </summary> public NumericTokenStream(AttributeSource source, int precisionStep) : base(source) { InitBlock(); this.precisionStep = precisionStep; if (precisionStep < 1) { throw new System.ArgumentException("precisionStep must be >=1"); } }
/// <summary> Re-initialize the state, using this boost value.</summary> /// <param name="docBoost">boost value to use. /// </param> internal void Reset(float docBoost) { position = 0; length = 0; numOverlap = 0; offset = 0; boost = docBoost; attributeSource = null; }
/// <summary> Re-initialize the state, using this boost value.</summary> /// <param name="docBoost">boost value to use. /// </param> internal void Reset(float docBoost) { position = 0; length = 0; numOverlap = 0; offset = 0; boost = docBoost; attributeSource = null; }
/// <summary> Returns true, iff the current state of the passed-in {@link AttributeSource} shall be stored /// in the sink. /// </summary> public abstract bool Accept(AttributeSource source);
/// <summary> A TokenStream that uses the same attributes as the supplied one.</summary> protected internal TokenStream(AttributeSource input):base(input) { InitBlock(); tokenWrapper = InitTokenWrapper(input); Check(); }
public override bool Accept(AttributeSource source) { return(true); }
public CharTokenizer(AttributeSource source, System.IO.TextReader input) : base(source, input) { offsetAtt = (OffsetAttribute)AddAttribute(typeof(OffsetAttribute)); termAtt = (TermAttribute)AddAttribute(typeof(TermAttribute)); }
/// <summary>Construct a token stream processing the given input using the given AttributeSource. </summary> protected internal Tokenizer(AttributeSource source, System.IO.TextReader input):base(source) { this.input = CharReader.Get(input); }
internal SinkTokenStream(AttributeSource source, SinkFilter filter) : base(source) { this.filter = filter; }
/// <summary> Creates a new StandardTokenizer with a given {@link AttributeSource}.</summary> public StandardTokenizer(Version matchVersion, AttributeSource source, System.IO.TextReader input):base(source) { InitBlock(); this.scanner = new StandardTokenizerImpl(input); Init(input, matchVersion); }
internal /*private*/ void AddState(AttributeSource.State state) { if (it != null) { throw new System.SystemException("The tee must be consumed before sinks are consumed."); } cachedStates.Add(state); }
internal /*private*/ bool Accept(AttributeSource source) { return filter.Accept(source); }
internal SinkTokenStream(AttributeSource source, SinkFilter filter):base(source) { this.filter = filter; }
/// <summary> Returns true, iff the current state of the passed-in {@link AttributeSource} shall be stored /// in the sink. /// </summary> public abstract bool Accept(AttributeSource source);
public KeywordTokenizer(AttributeSource source, System.IO.TextReader input, int bufferSize) : base(source, input) { Init(bufferSize); }
/// <summary>Construct a new WhitespaceTokenizer using a given {@link AttributeSource}. </summary> public WhitespaceTokenizer(AttributeSource source, System.IO.TextReader in_Renamed) : base(source, in_Renamed) { }
internal /*private*/ void SetFinalState(AttributeSource.State finalState) { this.finalState = finalState; }
public StandardTokenizer(AttributeSource source, System.IO.TextReader input, bool replaceInvalidAcronym):base(source) { InitBlock(); this.scanner = new StandardTokenizerImpl(input); Init(input, replaceInvalidAcronym); }
public override bool Accept(AttributeSource source) { return true; }
/// <summary> A TokenStream that uses the same attributes as the supplied one.</summary> protected internal TokenStream(AttributeSource input) : base(input) { InitBlock(); tokenWrapper = InitTokenWrapper(input); Check(); }
/// <summary>Construct a new LowerCaseTokenizer using a given {@link AttributeSource}. </summary> public LowerCaseTokenizer(AttributeSource source, System.IO.TextReader in_Renamed) : base(source, in_Renamed) { }
/// <summary>Construct a token stream processing the given input using the given AttributeSource. </summary> protected internal Tokenizer(AttributeSource source):base(source) { }
/// <summary>Construct a new LetterTokenizer using a given {@link AttributeSource}. </summary> public LetterTokenizer(AttributeSource source, System.IO.TextReader in_Renamed):base(source, in_Renamed) { }
/// <summary> Expert: Creates a token stream for numeric values with the specified /// <code>precisionStep</code> using the given {@link AttributeSource}. /// The stream is not yet initialized, /// before using set a value using the various set<em>???</em>Value() methods. /// </summary> public NumericTokenStream(AttributeSource source, int precisionStep):base(source) { InitBlock(); this.precisionStep = precisionStep; if (precisionStep < 1) throw new System.ArgumentException("precisionStep must be >=1"); }
/// <summary>Construct a token stream processing the given input using the given AttributeSource. </summary> protected internal Tokenizer(AttributeSource source) : base(source) { }
/// <summary>Construct a new WhitespaceTokenizer using a given {@link AttributeSource}. </summary> public WhitespaceTokenizer(AttributeSource source, System.IO.TextReader in_Renamed):base(source, in_Renamed) { }
internal /*private*/ bool Accept(AttributeSource source) { return(filter.Accept(source)); }
public CharTokenizer(AttributeSource source, System.IO.TextReader input):base(source, input) { offsetAtt = (OffsetAttribute) AddAttribute(typeof(OffsetAttribute)); termAtt = (TermAttribute) AddAttribute(typeof(TermAttribute)); }
public StandardTokenizer(AttributeSource source, System.IO.TextReader input, bool replaceInvalidAcronym) : base(source) { InitBlock(); this.scanner = new StandardTokenizerImpl(input); Init(input, replaceInvalidAcronym); }
/// <summary>Construct a token stream processing the given input using the given AttributeSource. </summary> protected internal Tokenizer(AttributeSource source, System.IO.TextReader input) : base(source) { this.input = CharReader.Get(input); }
/// <summary> Creates a new StandardTokenizer with a given {@link AttributeSource}.</summary> public StandardTokenizer(Version matchVersion, AttributeSource source, System.IO.TextReader input) : base(source) { InitBlock(); this.scanner = new StandardTokenizerImpl(input); Init(input, matchVersion); }
private TokenWrapper InitTokenWrapper(AttributeSource input) { if (onlyUseNewAPI) { // no wrapper needed return null; } else { // if possible get the wrapper from the filter's input stream if (input is TokenStream && ((TokenStream) input).tokenWrapper != null) { return ((TokenStream) input).tokenWrapper; } // check that all attributes are implemented by the same TokenWrapper instance Attribute att = AddAttribute(typeof(TermAttribute)); if (att is TokenWrapper && AddAttribute(typeof(TypeAttribute)) == att && AddAttribute(typeof(PositionIncrementAttribute)) == att && AddAttribute(typeof(FlagsAttribute)) == att && AddAttribute(typeof(OffsetAttribute)) == att && AddAttribute(typeof(PayloadAttribute)) == att) { return (TokenWrapper) att; } else { throw new System.NotSupportedException("If onlyUseNewAPI is disabled, all basic Attributes must be implemented by the internal class " + "TokenWrapper. Please make sure, that all TokenStreams/TokenFilters in this chain have been " + "instantiated with this flag disabled and do not add any custom instances for the basic Attributes!"); } } }
public KeywordTokenizer(AttributeSource source, System.IO.TextReader input, int bufferSize):base(source, input) { Init(bufferSize); }