private static bool TryTakeIdentifier(StringKeeper stringKeeper, out StringKeeper newStringKeeper, out Token token)
{
var work = new StringKeeper(stringKeeper);
if (CanStartIdentifier(work.Next))
{
work = work.Take();
while (ValidIdentifierChar(work.Next))
{
work = work.Take();
}
if (work.IsEmpty || work.IsWhitespace)
{
var difference = work.Difference(stringKeeper);
newStringKeeper = work;
token = new Token(TokenType.Identifier, difference);
return(true);
}
}
newStringKeeper = stringKeeper;
token = null;
return(false);
}
private static bool TryTakeStringLiteral(StringKeeper stringKeeper, out StringKeeper newStringKeeper, out Token token)
{
if (stringKeeper.Next == '\'')
{
var start = stringKeeper.Take();
var work = new StringKeeper(start);
while (!work.IsEmpty)
{
if (work.IsNext("\'\'"))
{
work = work.Take(2);
}
else
{
if (work.Next == '\'')
{
var literalValue = work.Difference(start).Replace("''", "'");
token = new Token(TokenType.StringLiteral, literalValue);
work = work.Take();
newStringKeeper = work;
if (!IsTerminator(work))
{
do
{
work = work.Take();
} while (!work.IsEmpty && !work.IsWhitespace);
token.TokenType = TokenType.Error;
return(true);
}
return(true);
}
else
{
work = work.Take();
}
}
}
newStringKeeper = work;
token = new Token(TokenType.Error, work.Difference(stringKeeper));
return(true);
}
newStringKeeper = stringKeeper;
token = null;
return(false);
}
public static IEnumerable <Token> Analyse(string text)
{
var stringKeeper = new StringKeeper(text);
while (!stringKeeper.IsEmpty)
{
StringKeeper newStringKeeper;
Token token;
if (stringKeeper.IsWhitespace)
{
stringKeeper = stringKeeper.Take();
}
else if (TryTakeParens(stringKeeper, out newStringKeeper, out token))
{
stringKeeper = newStringKeeper;
yield return(token);
}
else if (TryTakeOperator(stringKeeper, out newStringKeeper, out token))
{
stringKeeper = newStringKeeper;
yield return(token);
}
else if (TryTakeLiteral(stringKeeper, out newStringKeeper, out token))
{
stringKeeper = newStringKeeper;
yield return(token);
}
else if (TryTakeIdentifier(stringKeeper, out newStringKeeper, out token))
{
stringKeeper = newStringKeeper;
yield return(token);
}
}
}
private static bool TryTakeDateTime(StringKeeper stringKeeper, out StringKeeper newStringKeeper, out Token token)
{
if (stringKeeper.IsNext("datetime'"))
{
var work = stringKeeper.Take(8);
Debug.Assert(work.Next == '\'');
StringKeeper literalEnd;
Token literalToken;
TryTakeStringLiteral(work, out literalEnd, out literalToken);
newStringKeeper = literalEnd;
if (literalToken.TokenType == TokenType.Error)
{
token = new Token(TokenType.Error, literalEnd.Difference(stringKeeper));
return(true);
}
literalToken.TokenType = TokenType.DateTimeLiteral;
token = literalToken;
return(true);
}
newStringKeeper = stringKeeper;
token = null;
return(false);
}
private static bool TryTakeParens(StringKeeper stringKeeper, out StringKeeper newStringKeeper, out Token token)
{
var thisParens = _parens.FirstOrDefault(x => stringKeeper.IsNext(x.Key));
if (thisParens.Key != null)
{
var work = stringKeeper.Take(thisParens.Key.Length);
newStringKeeper = work;
token = thisParens.Value;
return(true);
}
newStringKeeper = stringKeeper;
token = null;
return(false);
}
private static bool TryTakeOperator(StringKeeper stringKeeper, out StringKeeper newStringKeeper, out Token token)
{
var thisOp = _operators.FirstOrDefault(x => stringKeeper.IsNext(x.Key));
if (thisOp.Key != null)
{
var work = stringKeeper.Take(thisOp.Key.Length);
if (work.IsEmpty || work.IsWhitespace || work.IsNext("("))
{
newStringKeeper = work;
token = thisOp.Value;
return(true);
}
}
newStringKeeper = stringKeeper;
token = null;
return(false);
}
private static bool TryTakeNumeric(StringKeeper stringKeeper, out StringKeeper newStringKeeper, out Token token)
{
if (char.IsDigit(stringKeeper.Next))
{
var invalidNumber = false;
var work = new StringKeeper(stringKeeper);
var doubleValue = false;
do
{
work = work.Take();
if (!work.IsEmpty && work.Next == '.' && !doubleValue)
{
doubleValue = true;
work = work.Take();
if (!char.IsDigit(work.Next))
{
invalidNumber = true;
}
}
} while (!work.IsEmpty && char.IsDigit(work.Next));
var value = work.Difference(stringKeeper);
var longInt = false;
if (!work.IsEmpty && work.Next == 'L')
{
longInt = true;
work = work.Take();
}
if (!IsTerminator(work))
{
do
{
work = work.Take();
} while (!work.IsWhitespace && !work.IsEmpty);
token = new Token(TokenType.Error, work.Difference(stringKeeper));
newStringKeeper = work;
return(true);
}
TokenType tokenType;
if (invalidNumber || (longInt && doubleValue))
{
tokenType = TokenType.Error;
}
else
{
if (doubleValue)
{
tokenType = TokenType.DoubleLiteral;
}
else
{
tokenType = longInt ? TokenType.LongLiteral : TokenType.IntLiteral;
}
}
token = new Token(tokenType, value);
newStringKeeper = work;
return(true);
}
newStringKeeper = stringKeeper;
token = null;
return(false);
}
