private TokenWrapper InitTokenWrapper(AttributeSource input)
{
if (onlyUseNewAPI)
{
// no wrapper needed
return(null);
}
else
{
// if possible get the wrapper from the filter's input stream
if (input is TokenStream && ((TokenStream)input).tokenWrapper != null)
{
return(((TokenStream)input).tokenWrapper);
}
// check that all attributes are implemented by the same TokenWrapper instance
Attribute att = AddAttribute(typeof(TermAttribute));
if (att is TokenWrapper && AddAttribute(typeof(TypeAttribute)) == att && AddAttribute(typeof(PositionIncrementAttribute)) == att && AddAttribute(typeof(FlagsAttribute)) == att && AddAttribute(typeof(OffsetAttribute)) == att && AddAttribute(typeof(PayloadAttribute)) == att)
{
return((TokenWrapper)att);
}
else
{
throw new System.NotSupportedException("If onlyUseNewAPI is disabled, all basic Attributes must be implemented by the internal class " + "TokenWrapper. Please make sure, that all TokenStreams/TokenFilters in this chain have been " + "instantiated with this flag disabled and do not add any custom instances for the basic Attributes!");
}
}
}
/// <summary> Expert: Creates a token stream for numeric values with the specified
/// <code>precisionStep</code> using the given {@link AttributeSource}.
/// The stream is not yet initialized,
/// before using set a value using the various set<em>???</em>Value() methods.
/// </summary>
public NumericTokenStream(AttributeSource source, int precisionStep) : base(source)
{
InitBlock();
this.precisionStep = precisionStep;
if (precisionStep < 1)
{
throw new System.ArgumentException("precisionStep must be >=1");
}
}
/// <summary> Re-initialize the state, using this boost value.</summary>
/// <param name="docBoost">boost value to use.
/// </param>
internal void Reset(float docBoost)
{
position = 0;
length = 0;
numOverlap = 0;
offset = 0;
boost = docBoost;
attributeSource = null;
}
/// <summary> Re-initialize the state, using this boost value.</summary>
/// <param name="docBoost">boost value to use.
/// </param>
internal void Reset(float docBoost)
{
position = 0;
length = 0;
numOverlap = 0;
offset = 0;
boost = docBoost;
attributeSource = null;
}
/// <summary> Returns true, iff the current state of the passed-in {@link AttributeSource} shall be stored
/// in the sink.
/// </summary>
public abstract bool Accept(AttributeSource source);
/// <summary> A TokenStream that uses the same attributes as the supplied one.</summary>
protected internal TokenStream(AttributeSource input):base(input)
{
InitBlock();
tokenWrapper = InitTokenWrapper(input);
Check();
}
public override bool Accept(AttributeSource source)
{
return(true);
}
public CharTokenizer(AttributeSource source, System.IO.TextReader input) : base(source, input)
{
offsetAtt = (OffsetAttribute)AddAttribute(typeof(OffsetAttribute));
termAtt = (TermAttribute)AddAttribute(typeof(TermAttribute));
}
/// <summary>Construct a token stream processing the given input using the given AttributeSource. </summary>
protected internal Tokenizer(AttributeSource source, System.IO.TextReader input):base(source)
{
this.input = CharReader.Get(input);
}
internal SinkTokenStream(AttributeSource source, SinkFilter filter) : base(source)
{
this.filter = filter;
}
/// <summary> Creates a new StandardTokenizer with a given {@link AttributeSource}.</summary>
public StandardTokenizer(Version matchVersion, AttributeSource source, System.IO.TextReader input):base(source)
{
InitBlock();
this.scanner = new StandardTokenizerImpl(input);
Init(input, matchVersion);
}
internal /*private*/ void AddState(AttributeSource.State state)
{
if (it != null)
{
throw new System.SystemException("The tee must be consumed before sinks are consumed.");
}
cachedStates.Add(state);
}
internal /*private*/ bool Accept(AttributeSource source)
{
return filter.Accept(source);
}
internal SinkTokenStream(AttributeSource source, SinkFilter filter):base(source)
{
this.filter = filter;
}
/// <summary> Returns true, iff the current state of the passed-in {@link AttributeSource} shall be stored
/// in the sink.
/// </summary>
public abstract bool Accept(AttributeSource source);
public KeywordTokenizer(AttributeSource source, System.IO.TextReader input, int bufferSize) : base(source, input)
{
Init(bufferSize);
}
/// <summary>Construct a new WhitespaceTokenizer using a given {@link AttributeSource}. </summary>
public WhitespaceTokenizer(AttributeSource source, System.IO.TextReader in_Renamed) : base(source, in_Renamed)
{
}
internal /*private*/ void SetFinalState(AttributeSource.State finalState)
{
this.finalState = finalState;
}
public StandardTokenizer(AttributeSource source, System.IO.TextReader input, bool replaceInvalidAcronym):base(source)
{
InitBlock();
this.scanner = new StandardTokenizerImpl(input);
Init(input, replaceInvalidAcronym);
}
public override bool Accept(AttributeSource source)
{
return true;
}
/// <summary> A TokenStream that uses the same attributes as the supplied one.</summary>
protected internal TokenStream(AttributeSource input) : base(input)
{
InitBlock();
tokenWrapper = InitTokenWrapper(input);
Check();
}
/// <summary>Construct a new LowerCaseTokenizer using a given {@link AttributeSource}. </summary>
public LowerCaseTokenizer(AttributeSource source, System.IO.TextReader in_Renamed) : base(source, in_Renamed)
{
}
/// <summary>Construct a token stream processing the given input using the given AttributeSource. </summary>
protected internal Tokenizer(AttributeSource source):base(source)
{
}
/// <summary>Construct a new LetterTokenizer using a given {@link AttributeSource}. </summary>
public LetterTokenizer(AttributeSource source, System.IO.TextReader in_Renamed):base(source, in_Renamed)
{
}
/// <summary> Expert: Creates a token stream for numeric values with the specified
/// <code>precisionStep</code> using the given {@link AttributeSource}.
/// The stream is not yet initialized,
/// before using set a value using the various set<em>???</em>Value() methods.
/// </summary>
public NumericTokenStream(AttributeSource source, int precisionStep):base(source)
{
InitBlock();
this.precisionStep = precisionStep;
if (precisionStep < 1)
throw new System.ArgumentException("precisionStep must be >=1");
}
/// <summary>Construct a token stream processing the given input using the given AttributeSource. </summary>
protected internal Tokenizer(AttributeSource source) : base(source)
{
}
/// <summary>Construct a new WhitespaceTokenizer using a given {@link AttributeSource}. </summary>
public WhitespaceTokenizer(AttributeSource source, System.IO.TextReader in_Renamed):base(source, in_Renamed)
{
}
internal /*private*/ bool Accept(AttributeSource source)
{
return(filter.Accept(source));
}
public CharTokenizer(AttributeSource source, System.IO.TextReader input):base(source, input)
{
offsetAtt = (OffsetAttribute) AddAttribute(typeof(OffsetAttribute));
termAtt = (TermAttribute) AddAttribute(typeof(TermAttribute));
}
public StandardTokenizer(AttributeSource source, System.IO.TextReader input, bool replaceInvalidAcronym) : base(source)
{
InitBlock();
this.scanner = new StandardTokenizerImpl(input);
Init(input, replaceInvalidAcronym);
}
/// <summary>Construct a token stream processing the given input using the given AttributeSource. </summary>
protected internal Tokenizer(AttributeSource source, System.IO.TextReader input) : base(source)
{
this.input = CharReader.Get(input);
}
/// <summary> Creates a new StandardTokenizer with a given {@link AttributeSource}.</summary>
public StandardTokenizer(Version matchVersion, AttributeSource source, System.IO.TextReader input) : base(source)
{
InitBlock();
this.scanner = new StandardTokenizerImpl(input);
Init(input, matchVersion);
}
private TokenWrapper InitTokenWrapper(AttributeSource input)
{
if (onlyUseNewAPI)
{
// no wrapper needed
return null;
}
else
{
// if possible get the wrapper from the filter's input stream
if (input is TokenStream && ((TokenStream) input).tokenWrapper != null)
{
return ((TokenStream) input).tokenWrapper;
}
// check that all attributes are implemented by the same TokenWrapper instance
Attribute att = AddAttribute(typeof(TermAttribute));
if (att is TokenWrapper && AddAttribute(typeof(TypeAttribute)) == att && AddAttribute(typeof(PositionIncrementAttribute)) == att && AddAttribute(typeof(FlagsAttribute)) == att && AddAttribute(typeof(OffsetAttribute)) == att && AddAttribute(typeof(PayloadAttribute)) == att)
{
return (TokenWrapper) att;
}
else
{
throw new System.NotSupportedException("If onlyUseNewAPI is disabled, all basic Attributes must be implemented by the internal class " + "TokenWrapper. Please make sure, that all TokenStreams/TokenFilters in this chain have been " + "instantiated with this flag disabled and do not add any custom instances for the basic Attributes!");
}
}
}
public KeywordTokenizer(AttributeSource source, System.IO.TextReader input, int bufferSize):base(source, input)
{
Init(bufferSize);
}
