protected override Token IsMatchImpl( Tokenizer tokenizer )
{
var str = new StringBuilder();
if( tokenizer.Current == StringDelim )
{
tokenizer.Consume();
while( !tokenizer.EndOfStream() && tokenizer.Current != StringDelim )
{
str.Append( tokenizer.Current );
tokenizer.Consume();
}
if( tokenizer.Current == StringDelim )
{
tokenizer.Consume();
}
}
if( str.Length > 0 )
{
return new Token( TokenType.QuotedString, str.ToString() );
}
return null;
}
protected override Token IsMatchImpl( Tokenizer tokenizer )
{
var str = new StringBuilder();
while( !tokenizer.EndOfStream() && String.IsNullOrWhiteSpace( tokenizer.Current ) )
{
str.Append( tokenizer.Current );
tokenizer.Consume();
}
if( str.Length > 0 )
{
return new Token( TokenType.WhiteSpace, str.ToString() );
}
return null;
}
protected override Token IsMatchImpl( Tokenizer tokenizer )
{
var str = new StringBuilder();
while( !tokenizer.EndOfStream() && ( tokenizer.Current == "\r" || tokenizer.Current == "\n" ) )
{
str.Append( tokenizer.Current );
tokenizer.Consume();
}
if( str.ToString() == Environment.NewLine )
{
return new Token( TokenType.NewLine );
}
return null;
}
public Token IsMatch( Tokenizer tokenizer )
{
if( tokenizer.EndOfStream() )
{
return new Token( TokenType.EndOfStream );
}
tokenizer.TakeSnapshot();
var match = IsMatchImpl( tokenizer );
if( match == null )
{
tokenizer.RollbackSnapshot();
}
else
{
tokenizer.CommitSnapshot();
}
return match;
}
protected override Token IsMatchImpl( Tokenizer tokenizer )
{
String current = null;
while( !tokenizer.EndOfStream() && !String.IsNullOrWhiteSpace( tokenizer.Current ) && mySpecialCharacters.All( m => m.Match != tokenizer.Current ) )
{
current += tokenizer.Current;
tokenizer.Consume();
}
if( current == null )
{
return null;
}
// can't start a word with a special character
if( mySpecialCharacters.Any( c => current.StartsWith( c.Match ) ) )
{
throw new Exception( String.Format( "Cannot start a word with a special character {0}", current ) );
}
return new Token( TokenType.Word, current );
}