public override bool Accept(AttributeSource a)
{
bool b = (a != null && count % modCount == 0);
count++;
return(b);
}
private TokenWrapper InitTokenWrapper(AttributeSource input)
{
if (onlyUseNewAPI)
{
// no wrapper needed
return(null);
}
else
{
// if possible get the wrapper from the filter's input stream
if (input is TokenStream && ((TokenStream)input).tokenWrapper != null)
{
return(((TokenStream)input).tokenWrapper);
}
// check that all attributes are implemented by the same TokenWrapper instance
Attribute att = AddAttribute(typeof(TermAttribute));
if (att is TokenWrapper && AddAttribute(typeof(TypeAttribute)) == att && AddAttribute(typeof(PositionIncrementAttribute)) == att && AddAttribute(typeof(FlagsAttribute)) == att && AddAttribute(typeof(OffsetAttribute)) == att && AddAttribute(typeof(PayloadAttribute)) == att)
{
return((TokenWrapper)att);
}
else
{
throw new System.NotSupportedException("If onlyUseNewAPI is disabled, all basic Attributes must be implemented by the internal class " + "TokenWrapper. Please make sure, that all TokenStreams/TokenFilters in this chain have been " + "instantiated with this flag disabled and do not add any custom instances for the basic Attributes!");
}
}
}
/// <summary>
/// Returns the related attributes. </summary>
public virtual AttributeSource Attributes()
{
if (Atts == null)
{
Atts = new AttributeSource();
}
return(Atts);
}
/// <summary>
/// Returns the related attributes. </summary>
public virtual AttributeSource Attributes()
{
if (Atts == null)
{
Atts = new AttributeSource();
}
return Atts;
}
/// <summary> Re-initialize the state, using this boost value.</summary>
/// <param name="docBoost">boost value to use.
/// </param>
internal void Reset(float docBoost)
{
position = 0;
length = 0;
numOverlap = 0;
offset = 0;
boost = docBoost;
attributeSource = null;
}
/// <summary> Expert: Creates a token stream for numeric values with the specified
/// <c>precisionStep</c> using the given <see cref="AttributeSource" />.
/// The stream is not yet initialized,
/// before using set a value using the various set<em>???</em>Value() methods.
/// </summary>
public NumericTokenStream(AttributeSource source, int precisionStep) : base(source)
{
InitBlock();
this.precisionStep = precisionStep;
if (precisionStep < 1)
{
throw new System.ArgumentException("precisionStep must be >=1");
}
}
/// <summary> Re-initialize the state, using this boost value.</summary>
/// <param name="docBoost">boost value to use.
/// </param>
internal void Reset(float docBoost)
{
position = 0;
length = 0;
numOverlap = 0;
offset = 0;
boost = docBoost;
attributeSource = null;
}
/// <summary>
/// Re-initialize the state
/// </summary>
internal void Reset()
{
position = 0;
length = 0;
numOverlap = 0;
offset = 0;
maxTermFrequency = 0;
uniqueTermCount = 0;
boost = 1.0f;
attributeSource = null;
}
/// <summary>
/// Re-initialize the state
/// </summary>
internal void Reset()
{
Position_Renamed = 0;
Length_Renamed = 0;
NumOverlap_Renamed = 0;
Offset_Renamed = 0;
MaxTermFrequency_Renamed = 0;
UniqueTermCount_Renamed = 0;
Boost_Renamed = 1.0f;
AttributeSource_Renamed = null;
}
/// <summary>
/// Re-initialize the state
/// </summary>
internal void Reset()
{
Position_Renamed = 0;
Length_Renamed = 0;
NumOverlap_Renamed = 0;
Offset_Renamed = 0;
MaxTermFrequency_Renamed = 0;
UniqueTermCount_Renamed = 0;
Boost_Renamed = 1.0f;
AttributeSource_Renamed = null;
}
/// <summary>Construct a token stream processing the given input using the given AttributeSource. </summary>
protected internal Tokenizer(AttributeSource source):base(source)
{
}
public CharTokenizer(AttributeSource source, System.IO.TextReader input) : base(source, input)
{
offsetAtt = (OffsetAttribute)AddAttribute(typeof(OffsetAttribute));
termAtt = (TermAttribute)AddAttribute(typeof(TermAttribute));
}
public override TermsEnum GetTermsEnum(Terms terms, AttributeSource atts)
{
return new SimpleAutomatonTermsEnum(this, terms.Iterator(null));
}
/// <summary>Construct a new LowerCaseTokenizer using a given {@link AttributeSource}. </summary>
public LowerCaseTokenizer(AttributeSource source, System.IO.TextReader in_Renamed) : base(source, in_Renamed)
{
}
public KeywordTokenizer(AttributeSource source, System.IO.TextReader input, int bufferSize) : base(source, input)
{
Init(bufferSize);
}
/// <summary> A TokenStream that uses the same attributes as the supplied one.</summary>
protected internal TokenStream(AttributeSource input):base(input)
{
InitBlock();
tokenWrapper = InitTokenWrapper(input);
Check();
}
/// <summary> Creates a new StandardTokenizer with a given {@link AttributeSource}. </summary>
public StandardTokenizer(AttributeSource source, System.IO.TextReader input, bool replaceInvalidAcronym)
: base(source)
{
InitBlock();
this.scanner = new StandardTokenizerImpl(input);
Init(input, replaceInvalidAcronym);
}
/// <summary> Creates a new StandardTokenizer with a given {@link AttributeSource}.</summary>
public StandardTokenizer(Version matchVersion, AttributeSource source, System.IO.TextReader input) : base(source)
{
InitBlock();
this.scanner = new StandardTokenizerImpl(input);
Init(input, matchVersion);
}
public StandardTokenizer(AttributeSource source, System.IO.TextReader input, bool replaceInvalidAcronym) : base(source)
{
InitBlock();
this.scanner = new StandardTokenizerImpl(input);
Init(input, replaceInvalidAcronym);
}
/// <summary>Construct a new WhitespaceTokenizer using a given <see cref="AttributeSource" />. </summary>
public WhitespaceTokenizer(AttributeSource source, System.IO.TextReader @in)
: base(source, @in)
{
}
protected CharTokenizer(AttributeSource source, System.IO.TextReader input):base(source, input)
{
offsetAtt = AddAttribute<IOffsetAttribute>();
termAtt = AddAttribute<ITermAttribute>();
}
/// <summary>Construct a new WhitespaceTokenizer using a given {@link AttributeSource}. </summary>
public WhitespaceTokenizer(AttributeSource source, System.IO.TextReader in_Renamed) : base(source, in_Renamed)
{
}
protected CharTokenizer(AttributeSource source, System.IO.TextReader input) : base(source, input)
{
offsetAtt = AddAttribute <IOffsetAttribute>();
termAtt = AddAttribute <ITermAttribute>();
}
public CharTokenizer(AttributeSource source, System.IO.TextReader input)
: base(source, input)
{
offsetAtt = (OffsetAttribute) AddAttribute(typeof(OffsetAttribute));
termAtt = (TermAttribute) AddAttribute(typeof(TermAttribute));
}
/// <summary> Creates a new StandardTokenizer with a given {@link AttributeSource}.</summary>
public StandardTokenizer(Version matchVersion, AttributeSource source, System.IO.TextReader input):base(source)
{
InitBlock();
this.scanner = new StandardTokenizerImpl(input);
Init(input, matchVersion);
}
/*private*/
internal bool Accept(AttributeSource source)
{
return filter.Accept(source);
}
/// <summary> /// Construct the enumeration to be used, expanding the /// pattern term. this method should only be called if /// the field exists (ie, implementations can assume the /// field does exist). this method should not return null /// (should instead return <seealso cref="TermsEnum#EMPTY"/> if no /// terms match). The TermsEnum must already be /// positioned to the first matching term. /// The given <seealso cref="AttributeSource"/> is passed by the <seealso cref="RewriteMethod"/> to /// provide attributes, the rewrite method uses to inform about e.g. maximum competitive boosts. /// this is currently only used by <seealso cref="TopTermsRewrite"/> /// </summary> public abstract TermsEnum GetTermsEnum(Terms terms, AttributeSource atts);
public KeywordTokenizer(AttributeSource source, System.IO.TextReader input, int bufferSize)
: base(source, input)
{
Init(bufferSize);
}
public override bool Accept(AttributeSource a)
{
TermAttribute termAtt = (TermAttribute) a.GetAttribute(typeof(TermAttribute));
return termAtt.Term().ToUpper().Equals("Dogs".ToUpper());
}
/// <summary> A TokenStream that uses the same attributes as the supplied one.</summary>
protected internal TokenStream(AttributeSource input) : base(input)
{
InitBlock();
tokenWrapper = InitTokenWrapper(input);
Check();
}
protected override TermsEnum GetTermsEnum(Terms terms, AttributeSource atts)
{
return new SimpleAutomatonTermsEnum(this, terms.GetEnumerator());
}
public override bool Accept(AttributeSource a)
{
ITermAttribute termAtt = a.GetAttribute<ITermAttribute>();
return termAtt.Term.ToUpper().Equals("Dogs".ToUpper());
}
public override TermsEnum GetTermsEnum(Terms terms, AttributeSource atts)
{
return new TermRangeTermsEnumAnonymousInnerClassHelper(this, terms.Iterator(null), new BytesRef("2"), new BytesRef("7"));
}
/// <summary>Construct a token stream processing the given input using the given AttributeSource. </summary>
protected internal Tokenizer(AttributeSource source) : base(source)
{
}
/// <summary> Expert: Creates a token stream for numeric values with the specified
/// <c>precisionStep</c> using the given <see cref="AttributeSource" />.
/// The stream is not yet initialized,
/// before using set a value using the various set<em>???</em>Value() methods.
/// </summary>
public NumericTokenStream(AttributeSource source, int precisionStep)
: base(source)
{
InitBlock();
this.precisionStep = precisionStep;
if (precisionStep < 1)
throw new System.ArgumentException("precisionStep must be >=1");
}
public override bool Accept(AttributeSource a)
{
TermAttribute termAtt = (TermAttribute)a.GetAttribute(typeof(TermAttribute));
return(termAtt.Term().ToUpper().Equals("Dogs".ToUpper()));
}
/// <summary>Construct a new LetterTokenizer using a given <see cref="AttributeSource" />. </summary>
public LetterTokenizer(AttributeSource source, System.IO.TextReader @in)
: base(source, @in)
{
}
internal SinkTokenStream(AttributeSource source, SinkFilter filter)
: base(source)
{
this.filter = filter;
}
/// <summary>Construct a token stream processing the given input using the given AttributeSource. </summary>
protected internal Tokenizer(AttributeSource source, System.IO.TextReader input):base(source)
{
this.input = CharReader.Get(input);
}
internal /*private*/ void AddState(AttributeSource.State state)
{
if (it != null)
{
throw new System.SystemException("The tee must be consumed before sinks are consumed.");
}
cachedStates.AddLast(state);
}
/// <summary>Construct a new WhitespaceTokenizer using a given {@link AttributeSource}. </summary>
public WhitespaceTokenizer(AttributeSource source, System.IO.TextReader in_Renamed):base(source, in_Renamed)
{
}
public override bool Accept(AttributeSource source)
{
return true;
}
public override TermsEnum GetTermsEnum(Terms terms, AttributeSource atts)
{
return new SimplePrefixTermsEnum(this, terms.Iterator(null), Prefix);
}
public override bool Accept(AttributeSource a)
{
bool b = (a != null && count % modCount == 0);
count++;
return b;
}
/// <summary>Construct a token stream processing the given input using the given AttributeSource. </summary>
protected internal Tokenizer(AttributeSource source, System.IO.TextReader input) : base(source)
{
this.input = CharReader.Get(input);
}
/// <summary>
/// Returns the <seealso cref="MultiTermQuery"/>s <seealso cref="TermsEnum"/> </summary>
/// <seealso cref= MultiTermQuery#getTermsEnum(Terms, AttributeSource) </seealso>
protected internal virtual TermsEnum GetTermsEnum(MultiTermQuery query, Terms terms, AttributeSource atts)
{
return(query.GetTermsEnum(terms, atts)); // allow RewriteMethod subclasses to pull a TermsEnum from the MTQ
}
/// <summary> Returns true, iff the current state of the passed-in <see cref="AttributeSource" /> shall be stored /// in the sink. /// </summary> public abstract bool Accept(AttributeSource source);
/// <summary> Returns true, iff the current state of the passed-in {@link AttributeSource} shall be stored
/// in the sink.
/// </summary>
public abstract bool Accept(AttributeSource source);
internal /*private*/ bool Accept(AttributeSource source)
{
return filter.Accept(source);
}
internal SinkTokenStream(AttributeSource source, SinkFilter filter) : base(source)
{
this.filter = filter;
}
internal /*private*/ void SetFinalState(AttributeSource.State finalState)
{
this.finalState = finalState;
}
internal /*private*/ bool Accept(AttributeSource source)
{
return(filter.Accept(source));
}
/// <summary>Construct a new LowerCaseTokenizer using a given <see cref="AttributeSource" />. </summary>
public LowerCaseTokenizer(AttributeSource source, System.IO.TextReader @in)
: base(source, @in)
{
}
public override bool Accept(AttributeSource source)
{
return(true);
}
public override bool Accept(AttributeSource a)
{
ITermAttribute termAtt = a.GetAttribute <ITermAttribute>();
return(termAtt.Term.ToUpper().Equals("Dogs".ToUpper()));
}
/// <summary>Construct a new LowerCaseTokenizer using a given {@link AttributeSource}. </summary>
public LowerCaseTokenizer(AttributeSource source, System.IO.TextReader in_Renamed):base(source, in_Renamed)
{
}
private TokenWrapper InitTokenWrapper(AttributeSource input)
{
if (onlyUseNewAPI)
{
// no wrapper needed
return null;
}
else
{
// if possible get the wrapper from the filter's input stream
if (input is TokenStream && ((TokenStream) input).tokenWrapper != null)
{
return ((TokenStream) input).tokenWrapper;
}
// check that all attributes are implemented by the same TokenWrapper instance
Attribute att = AddAttribute(typeof(TermAttribute));
if (att is TokenWrapper && AddAttribute(typeof(TypeAttribute)) == att && AddAttribute(typeof(PositionIncrementAttribute)) == att && AddAttribute(typeof(FlagsAttribute)) == att && AddAttribute(typeof(OffsetAttribute)) == att && AddAttribute(typeof(PayloadAttribute)) == att)
{
return (TokenWrapper) att;
}
else
{
throw new System.NotSupportedException("If onlyUseNewAPI is disabled, all basic Attributes must be implemented by the internal class " + "TokenWrapper. Please make sure, that all TokenStreams/TokenFilters in this chain have been " + "instantiated with this flag disabled and do not add any custom instances for the basic Attributes!");
}
}
}
/// <summary>Construct a new WhitespaceTokenizer using a given <see cref="AttributeSource" />. </summary>
public WhitespaceTokenizer(AttributeSource source, System.IO.TextReader @in)
: base(source, @in)
{
}
