public IEnumerable <BQueryToken> GetTokens()
{
while (!this.textSource.IsEnded)
{
char c = this.textSource.Current;
if (c == '.')
{
yield return(new BQueryToken(BQueryTokenKind.Dot, ".", this.textSource.GetPosition()));
}
else if (c == '[')
{
yield return(new BQueryToken(BQueryTokenKind.LBracket, "[", this.textSource.GetPosition()));
}
else if (c == ']')
{
yield return(new BQueryToken(BQueryTokenKind.RBracket, "]", this.textSource.GetPosition()));
}
else
{
if (char.IsLetter(c))
{
var(ident, offset) = LexerHelper.GetAnyName(this.textSource.Slice());
yield return(new BQueryToken(BQueryTokenKind.Id, ident, this.textSource.GetPosition()));
this.textSource.AdvanceChar(offset);
}
else if (char.IsDigit(c))
{
var(numberString, offset) = LexerHelper.GetNumber(this.textSource.Slice());
yield return(new BQueryToken(BQueryTokenKind.Number, numberString, this.textSource.GetPosition()));
this.textSource.AdvanceChar(offset);
}
else if (c == '\'')
{
var(str, offset) = LexerHelper.GetSingleQuoteString(this.textSource.Slice());
yield return(new BQueryToken(BQueryTokenKind.String, str, this.textSource.GetPosition()));
this.textSource.AdvanceChar(offset);
}
else if (c == '\"')
{
var(str, offset) = LexerHelper.GetDoubleQuoteString(this.textSource.Slice());
yield return(new BQueryToken(BQueryTokenKind.String, str, this.textSource.GetPosition()));
this.textSource.AdvanceChar(offset);
}
}
this.textSource.AdvanceChar();
}
}
public IEnumerable <LispToken> GetTokens()
{
HashSet <char> charsSymbols = new HashSet <char>()
{
'+', '*', '/', //math
'=', '<', '>' //boolean
};
while (!this.textSource.IsEnded)
{
var c = this.textSource.Current;
if (c == ';')
{
//comment ignore until new line
do
{
this.textSource.AdvanceChar();
} while (!this.textSource.IsEnded && this.textSource.Current != '\n');
}
else if (c == '(')
{
yield return(new LispToken(LispTokenKind.LParen, "(", this.textSource.GetPosition()));
}
else if (c == ')')
{
yield return(new LispToken(LispTokenKind.RParen, ")", this.textSource.GetPosition()));
}
else if (c == '\'')
{
yield return(new LispToken(LispTokenKind.Quote, "'", this.textSource.GetPosition()));
}
else if (charsSymbols.Contains(c))
{
yield return(new LispToken(LispTokenKind.Symbol, c.ToString(), this.textSource.GetPosition()));
}
else if (c == '-')
{
if (char.IsDigit(this.textSource.Next))
{
this.textSource.AdvanceChar(); // for the '-' symbol
var(numberString, offset) = LexerHelper.GetNumber(this.textSource.Slice());
yield return(new LispToken(LispTokenKind.Number, '-' + numberString, this.textSource.GetPosition()));
this.textSource.AdvanceChar(offset);
}
else
{
yield return(new LispToken(LispTokenKind.Symbol, "-", this.textSource.GetPosition()));
}
}
else
{
if (char.IsLetter(c))
{
var match = Regex.Match(this.textSource.Slice().ToString(), @"[\w!_]+", RegexOptions.Compiled);
yield return(new LispToken(LispTokenKind.Symbol, match.Value, this.textSource.GetPosition()));
this.textSource.AdvanceChar(match.Length - 1);
}
else if (char.IsDigit(c))
{
var(numberString, offset) = LexerHelper.GetNumber(this.textSource.Slice());
yield return(new LispToken(LispTokenKind.Number, numberString, this.textSource.GetPosition()));
this.textSource.AdvanceChar(offset);
}
else if (c == '\"')
{
var(str, offset) = LexerHelper.GetDoubleQuoteString(this.textSource.Slice());
yield return(new LispToken(LispTokenKind.String, str, this.textSource.GetPosition()));
this.textSource.AdvanceChar(offset);
}
}
this.textSource.AdvanceChar();
}
}
public IEnumerable <CmdToken> GetTokens()
{
while (!this.textSource.IsEnded)
{
var currentChar = this.textSource.Current;
if (currentChar == '=')
{
var nextChar = this.textSource.Next;
if (nextChar == '=')
{
yield return(new CmdToken(CmdTokenKind.CompareEqual, "=="));
this.textSource.AdvanceChar();
}
else
{
yield return(new CmdToken(CmdTokenKind.EqualSign, "="));
}
}
else if (currentChar == '<')
{
var nextChar = this.textSource.Next;
if (nextChar == '=')
{
yield return(new CmdToken(CmdTokenKind.LessThanEqual, "<="));
this.textSource.AdvanceChar();
}
else
{
yield return(new CmdToken(CmdTokenKind.LessThan, "<"));
}
}
else if (currentChar == '>')
{
var nextChar = this.textSource.Next;
if (nextChar == '=')
{
yield return(new CmdToken(CmdTokenKind.GreaterThanEqual, ">="));
this.textSource.AdvanceChar();
}
else
{
yield return(new CmdToken(CmdTokenKind.GreaterThan, ">"));
}
}
else if (currentChar == '+')
{
yield return(new CmdToken(CmdTokenKind.Plus, "+"));
}
else if (currentChar == '-')
{
yield return(new CmdToken(CmdTokenKind.Minus, "-"));
}
else if (currentChar == '/')
{
yield return(new CmdToken(CmdTokenKind.Slash, "/"));
}
else if (currentChar == '*')
{
yield return(new CmdToken(CmdTokenKind.Star, "*"));
}
else if (currentChar == '(')
{
yield return(new CmdToken(CmdTokenKind.LParen, "("));
}
else if (currentChar == ')')
{
yield return(new CmdToken(CmdTokenKind.RParen, ")"));
}
else if (currentChar == ',')
{
yield return(new CmdToken(CmdTokenKind.Comma, ","));
}
else if (currentChar == '.')
{
yield return(new CmdToken(CmdTokenKind.Dot, "."));
}
else if (currentChar == '!')
{
var nextChar = this.textSource.Next;
if (nextChar == '=')
{
yield return(new CmdToken(CmdTokenKind.CompareNotEqual, "!="));
this.textSource.AdvanceChar();
}
}
else
{
if (char.IsLetter(currentChar))
{
var(ident, offset) = LexerHelper.GetAnyName(this.textSource.Slice());
yield return(MakeToken(ident));
this.textSource.AdvanceChar(offset);
}
else if (char.IsDigit(currentChar))
{
var(numberString, offset) = LexerHelper.GetNumber(this.textSource.Slice());
yield return(new CmdToken(CmdTokenKind.Number, numberString));
this.textSource.AdvanceChar(offset);
}
else if (currentChar == '\'')
{
var(str, offset) = LexerHelper.GetSingleQuoteString(this.textSource.Slice());
yield return(new CmdToken(CmdTokenKind.String, str));
this.textSource.AdvanceChar(offset);
}
else if (currentChar == '\"')
{
var(str, offset) = LexerHelper.GetDoubleQuoteString(this.textSource.Slice());
yield return(new CmdToken(CmdTokenKind.String, str));
this.textSource.AdvanceChar(offset);
}
}
this.textSource.AdvanceChar();
}
}