public void MultiplePeriodsPositionTest()
{
var lexer = Lexer.CreateState("1.2.3");
_ = NumberScanner.Scan(lexer, ref this.token);
Assert.Equal(3, lexer.Src.Index);
}
public void DoublePeriodTest()
{
var lexer = Lexer.CreateState("1.2.3");
bool result = NumberScanner.Scan(lexer, ref this.token);
Assert.False(result);
}
public void SuccessPositionTest()
{
var lexer = Lexer.CreateState("123");
_ = NumberScanner.Scan(lexer, ref this.token);
Assert.Equal(2, lexer.Src.Index);
}
public void StartsWithPeriodTest()
{
var lexer = Lexer.CreateState(".1");
bool result = NumberScanner.Scan(lexer, ref this.token);
Assert.True(result);
}
public void FloatTypeTest()
{
var lexer = Lexer.CreateState("1.23");
_ = NumberScanner.Scan(lexer, ref this.token);
Assert.Equal(TokenType.Float, this.token.Type);
}
public void FloatTest()
{
var lexer = Lexer.CreateState("1.23");
bool result = NumberScanner.Scan(lexer, ref this.token);
Assert.True(result);
}
public void EndsWithPeriodSpacePositionTest()
{
var lexer = Lexer.CreateState("1. ");
_ = NumberScanner.Scan(lexer, ref this.token);
Assert.Equal(1, lexer.Src.Index);
}
/// <summary>
/// Used to analyse the current textwindow and attempt to build
/// proper syntax tokens which will consist of constructs such as
/// identifiers, keywords, operators
/// Returns after generating a single token
/// </summary>
/// <returns></returns>
private SyntaxToken LexSyntaxToken()
{
// Break early
if (!TextWindow.HasCharactersLeftToProcess())
{
return(new SyntaxToken(SyntaxKind.EOFNode, ""));
}
// Intermediate var
char xcNextChar = TextWindow.PeekCharacter();
//
switch (xcNextChar)
{
// Literals need to be caught first
case '\'': // ' right case
case '\"': // " wrong case - needs to be fixed
// Scan ahead to see if the literal has been closed properly
SyntaxToken oLiteralToken;
if (LiteralScanner.ScanLiteral(TextWindow, out oLiteralToken))
{
return(oLiteralToken);
}
else
{
Debug.Write("Invalid literal token? (" + TextWindow.PopCharacter(9999) + ")");
// Problem.. return a dead token
return(new SyntaxToken(
SyntaxKind.UnknownNode,
Convert.ToString(TextWindow.PopCharacter())));
}
// Brackets
case '(':
return(new SyntaxToken(
SyntaxKind.OpenParenthesisToken,
Convert.ToString(TextWindow.PopCharacter())));
case ')':
return(new SyntaxToken(
SyntaxKind.CloseParenthesisToken,
Convert.ToString(TextWindow.PopCharacter())));
// SemiColon
case ';':
return(new SyntaxToken(
SyntaxKind.SemiColonToken,
Convert.ToString(TextWindow.PopCharacter())));
// Equality/Comparison Operators
case '=':
// Intermediate var
SyntaxKind eTemp = SyntaxKind.EqualsToken;
// If we really have an equals equals token
if (TextWindow.PeekCharacter(1) == '=')
{
eTemp = SyntaxKind.EqualsEqualsToken;
}
// Or an Greater than or equal token
else if (TextWindow.PeekCharacter(1) == '>')
{
eTemp = SyntaxKind.GreaterThanOrEqualToken;
}
// Or a less than or equal token
else if (TextWindow.PeekCharacter(1) == '<')
{
eTemp = SyntaxKind.LessThanOrEqualToToken;
}
// Move ahead a single char now
String sResult = Convert.ToString(TextWindow.PopCharacter());
// Move ahead again if we have a compound token
if (eTemp != SyntaxKind.EqualsToken)
{
sResult += TextWindow.PopCharacter();
}
// Return something
return(new SyntaxToken(
eTemp,
sResult));
// These two can be compound
case '<':
// If we really have an equals equals token
if (TextWindow.PeekCharacter(1) == '=')
{
// Return something
return(new SyntaxToken(
SyntaxKind.LessThanOrEqualToToken,
TextWindow.PopCharacter(2)));
}
// If we really have an equals equals token
if (TextWindow.PeekCharacter(1) == '>')
{
// Return something
return(new SyntaxToken(
SyntaxKind.DiamondToken,
TextWindow.PopCharacter(2)));
}
else
{
// Return something
return(new SyntaxToken(
SyntaxKind.LessThanToken,
Convert.ToString(TextWindow.PopCharacter())));
}
case '>':
// If we really have an equals equals token
if (TextWindow.PeekCharacter(1) == '=')
{
// Return something
return(new SyntaxToken(
SyntaxKind.GreaterThanOrEqualToken,
TextWindow.PopCharacter(2)));
}
else
{
// Return something
return(new SyntaxToken(
SyntaxKind.GreaterThanToken,
Convert.ToString(TextWindow.PopCharacter())));
}
// .
case '.':
// .. should be handled here in its own individual case
if (TextWindow.PeekCharacter(1) == '.')
{
return(new SyntaxToken(
SyntaxKind.DotDotToken,
TextWindow.PopCharacter(2)));
}
// Else return normal
return(new SyntaxToken(
SyntaxKind.DotToken,
Convert.ToString(TextWindow.PopCharacter())));
case ',':
return(new SyntaxToken(
SyntaxKind.CommaToken,
Convert.ToString(TextWindow.PopCharacter())));
// Hardest cases
case '*':
// Did we just find an orphaned */?
if (TextWindow.PeekCharacter(1) == '/')
{
// This could be an issue
return(new SyntaxToken(
SyntaxKind.StarSlashToken,
TextWindow.PopCharacter(2)));
}
// Is this a star token or a multiplier?
return(new SyntaxToken(
SyntaxKind.StarToken,
Convert.ToString(TextWindow.PopCharacter())));
case '-':
return(new SyntaxToken(SyntaxKind.MinusToken, Convert.ToString(TextWindow.PopCharacter())));
case '+':
return(new SyntaxToken(SyntaxKind.PlusToken, Convert.ToString(TextWindow.PopCharacter())));
case '/':
return(new SyntaxToken(SyntaxKind.SlashToken, Convert.ToString(TextWindow.PopCharacter())));
case '|':
// || (Concatinate)
if (TextWindow.PeekCharacter(1) == '|')
{
// This could be an issue
return(new SyntaxToken(
SyntaxKind.BarBarToken,
TextWindow.PopCharacter(2)));
}
else
{
// Convert into a double bar
// Standalone | symbol
return(new SyntaxToken(
SyntaxKind.BarBarToken,
Convert.ToString(TextWindow.PopCharacter(1)) + "|"));
}
// Ampersand is allowed too (For SAS Intable)
case '&':
// Invalid character (MUST TRAIL an _)
break;
// Alphanumeric are possible cases for an identifier
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'g':
case 'h':
case 'i':
case 'j':
case 'k':
case 'l':
case 'm':
case 'n':
case 'o':
case 'p':
case 'q':
case 'r':
case 's':
case 't':
case 'u':
case 'v':
case 'w':
case 'x':
case 'y':
case 'z':
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
case 'G':
case 'H':
case 'I':
case 'J':
case 'K':
case 'L':
case 'M':
case 'N':
case 'O':
case 'P':
case 'Q':
case 'R':
case 'S':
case 'T':
case 'U':
case 'V':
case 'W':
case 'X':
case 'Y':
case 'Z':
case '_':
SyntaxToken oReturnToken;
// Scan for any keyword or identifier
if (IdentifierOrKeywordScanner.ScanKeywordOrIdentifier(TextWindow, out oReturnToken))
{
return(oReturnToken);
}
else
{
Debug.Write("Invalid identifier token? (" + TextWindow.PeekCharacter() + ")");
// Problem.. return a dead token
return(new SyntaxToken(
SyntaxKind.UnknownNode,
Convert.ToString(TextWindow.PopCharacter())));
}
// Number Token
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
SyntaxToken oNumberToken;
// Scan for a Number/Integer
if (NumberScanner.ScanNumber(TextWindow, out oNumberToken))
{
return(oNumberToken);
}
else
{
Debug.Write("Invalid numeric token? (" + TextWindow.PeekCharacter() + ")");
// Problem.. return a dead token
return(new SyntaxToken(
SyntaxKind.UnknownNode,
Convert.ToString(TextWindow.PopCharacter())));
}
//
default:
// Try anything we possibly can here
// Use the cumbersome scans to figure out what token we have
break;
}
return(null);
}
public void ScanNullTest()
{
Assert.Throws <ArgumentNullException>(() =>
NumberScanner.Scan(null, ref this.token));
}
