public void handleEscapeCharacterInsideStringLiteral()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner("\"Word\\n\""));
dynamic stringToken = this.tokenScanner.readNextToken();
Assert.Equal("Word\\n", stringToken.getLexeme());
}
public void Parse(NumericToken numeric, ITokenScanner tokenScanner, CharacterMapBuilder builder, bool isLenientParsing)
{
for (var i = 0; i < numeric.Int; i++)
{
if (!tokenScanner.MoveNext() || !(tokenScanner.CurrentToken is HexToken inputCode))
{
throw new InvalidOperationException($"Base font characters definition contains invalid item at index {i}: {tokenScanner.CurrentToken}");
}
if (!tokenScanner.MoveNext())
{
throw new InvalidOperationException($"Base font characters definition contains invalid item at index {i}: {tokenScanner.CurrentToken}");
}
if (tokenScanner.CurrentToken is NameToken characterName)
{
builder.AddBaseFontCharacter(inputCode.Bytes, characterName.Data);
}
else if (tokenScanner.CurrentToken is HexToken characterCode)
{
builder.AddBaseFontCharacter(inputCode.Bytes, characterCode.Bytes);
}
else
{
throw new InvalidOperationException($"Base font characters definition contains invalid item at index {i}: {tokenScanner.CurrentToken}");
}
}
}
public void readKeywordPrintBeforeQuote()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner("print\"Hello World!\";"));
dynamic printToken = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.KEYWORD_PRINT, printToken.getType());
}
public void readBoolKeyword()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner("bool"));
dynamic token = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.TYPE_IDENTIFIER_BOOL, token.getType());
}
public void readAssertKeyword()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner("assert"));
dynamic token = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.KEYWORD_ASSERT, token.getType());
}
public void readStringKeyword()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner("string"));
dynamic token = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.TYPE_IDENTIFIER_STRING, token.getType());
}
public void readIntKeyword()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner("int"));
dynamic token = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.TYPE_IDENTIFIER_INTEGER, token.getType());
}
public void readReadKeyword()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner("read"));
dynamic token = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.KEYWORD_READ, token.getType());
}
public void readKeywordAssertBeforeParenthesis()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner("assert(x = 1);"));
dynamic assertToken = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.KEYWORD_ASSERT, assertToken.getType());
}
public void readMultipleSingleCharTokens()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner(";;;"));
Assert.Equal(MiniPLTokenType.SEMICOLON, this.tokenScanner.readNextToken().getType());
Assert.Equal(MiniPLTokenType.SEMICOLON, this.tokenScanner.readNextToken().getType());
Assert.Equal(MiniPLTokenType.SEMICOLON, this.tokenScanner.readNextToken().getType());
}
public void Parse(NumericToken numeric, ITokenScanner tokenScanner, CharacterMapBuilder builder, bool isLenientParsing)
{
for (var i = 0; i < numeric.Int; i++)
{
if (!tokenScanner.MoveNext() || !(tokenScanner.CurrentToken is HexToken inputCode))
{
if (tokenScanner.CurrentToken is OperatorToken op &&
(string.Equals(op.Data, "endbfchar", StringComparison.OrdinalIgnoreCase) ||
string.Equals(op.Data, "endcmap", StringComparison.OrdinalIgnoreCase)))
{
return;
}
throw new InvalidOperationException($"Base font characters definition contains invalid item at index {i}: {tokenScanner.CurrentToken}");
}
if (!tokenScanner.MoveNext())
{
throw new InvalidOperationException($"Base font characters definition contains invalid item at index {i}: {tokenScanner.CurrentToken}");
}
if (tokenScanner.CurrentToken is NameToken characterName)
{
builder.AddBaseFontCharacter(inputCode.Bytes, characterName.Data);
}
else if (tokenScanner.CurrentToken is HexToken characterCode)
{
builder.AddBaseFontCharacter(inputCode.Bytes, characterCode.Bytes);
}
else
{
throw new InvalidOperationException($"Base font characters definition contains invalid item at index {i}: {tokenScanner.CurrentToken}");
}
}
}
public void shouldBeAbleToReadComplexExpressionWithOrWithoutWhitespace(String source)
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner(source));
dynamic firstVIdentifier = this.tokenScanner.readNextToken();
dynamic assignmentOperator = this.tokenScanner.readNextToken();
dynamic leftParenthesis = this.tokenScanner.readNextToken();
dynamic jIdentifier = this.tokenScanner.readNextToken();
dynamic plusOperator = this.tokenScanner.readNextToken();
dynamic lastVIdentifier = this.tokenScanner.readNextToken();
dynamic rightParenthesis = this.tokenScanner.readNextToken();
dynamic firstMultiplication = this.tokenScanner.readNextToken();
dynamic xIdentifier = this.tokenScanner.readNextToken();
dynamic secondMultiplication = this.tokenScanner.readNextToken();
dynamic iIdentifier = this.tokenScanner.readNextToken();
dynamic semicolon = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.IDENTIFIER, firstVIdentifier.getType());
Assert.Equal(MiniPLTokenType.ASSIGNMENT_OPERATOR, assignmentOperator.getType());
Assert.Equal(MiniPLTokenType.LEFT_PARENTHESIS, leftParenthesis.getType());
Assert.Equal(MiniPLTokenType.IDENTIFIER, jIdentifier.getType());
Assert.Equal(MiniPLTokenType.PLUS, plusOperator.getType());
Assert.Equal(MiniPLTokenType.IDENTIFIER, lastVIdentifier.getType());
Assert.Equal(MiniPLTokenType.RIGHT_PARENTHESIS, rightParenthesis.getType());
Assert.Equal(MiniPLTokenType.ASTERISK, firstMultiplication.getType());
Assert.Equal(MiniPLTokenType.IDENTIFIER, xIdentifier.getType());
Assert.Equal(MiniPLTokenType.ASTERISK, secondMultiplication.getType());
Assert.Equal(MiniPLTokenType.IDENTIFIER, iIdentifier.getType());
Assert.Equal(MiniPLTokenType.SEMICOLON, semicolon.getType());
}
public void Parse(NumericToken numeric, ITokenScanner scanner, CharacterMapBuilder builder, bool isLenientParsing)
{
for (var i = 0; i < numeric.Int; i++)
{
if (!scanner.TryReadToken(out HexToken startHexToken))
{
throw new InvalidFontFormatException("Could not find the starting hex token for the CIDRange in this font.");
}
if (!scanner.TryReadToken(out HexToken endHexToken))
{
throw new InvalidFontFormatException("Could not find the end hex token for the CIDRange in this font.");
}
if (!scanner.TryReadToken(out NumericToken mappedCode))
{
throw new InvalidFontFormatException("Could not find the starting CID numeric token for the CIDRange in this font.");
}
var start = HexToken.ConvertHexBytesToInt(startHexToken);
var end = HexToken.ConvertHexBytesToInt(endHexToken);
var range = new CidRange(start, end, mappedCode.Int);
builder.AddCidRange(range);
}
}
public T Parse(ITokenScanner tokenScanner, ITokenMatcher tokenMatcher)
{
tokenMatcher.Reset();
astBuilder.Reset();
var context = new ParserContext
{
TokenScanner = tokenScanner,
TokenMatcher = tokenMatcher,
TokenQueue = new Queue <Token>(),
Errors = new List <ParserException>()
};
StartRule(context, RuleType.CucumberExpression);
int state = 0;
Token token;
do
{
token = ReadToken(context);
state = MatchToken(state, token, context);
} while(!token.IsEOF);
EndRule(context, RuleType.CucumberExpression);
if (context.Errors.Count > 0)
{
throw new CompositeParserException(context.Errors.ToArray());
}
return(GetResult(context));
}
public void keywordsCanBeReadWhenThereIsWhitespaceAfterwards(String source, MiniPLTokenType type)
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner(source));
dynamic token = this.tokenScanner.readNextToken();
Assert.Equal(type, token.getType());
}
public void Parse(NumericToken numeric, ITokenScanner tokenScanner, CharacterMapBuilder builder, bool isLenientParsing)
{
/*
* For example:
* 3 begincodespacerange
* <00> <80>
* <8140> <9ffc>
* <a0> <de>
* endcodespacerange
*/
var ranges = new List <CodespaceRange>(numeric.Int);
for (var i = 0; i < numeric.Int; i++)
{
if (!tokenScanner.MoveNext() || !(tokenScanner.CurrentToken is HexToken start))
{
throw new InvalidOperationException("Codespace range contains an unexpected token: " + tokenScanner.CurrentToken);
}
if (!tokenScanner.MoveNext() || !(tokenScanner.CurrentToken is HexToken end))
{
throw new InvalidOperationException("Codespace range contains an unexpected token: " + tokenScanner.CurrentToken);
}
ranges.Add(new CodespaceRange(start.Bytes, end.Bytes));
}
builder.CodespaceRanges = ranges;
}
public void unrecognizedSourceShouldReturnInvalidTokenWithTextAsToken(String source)
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner(source));
dynamic invalidToken = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.INVALID_TOKEN, invalidToken.getType());
Assert.Equal(source, invalidToken.getLexeme());
}
public void readsSingleCharacterTokenFromSource(String source, MiniPLTokenType type)
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner(source));
dynamic token = this.tokenScanner.readNextToken();
Assert.Equal(type, token.getType());
Assert.Equal(source, token.getLexeme());
}
public void rangeOperatorShouldBeRecognized()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner(".."));
dynamic rangeOperatorToken = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.RANGE_OPERATOR, rangeOperatorToken.getType());
Assert.Equal("..", rangeOperatorToken.getLexeme());
}
public void assignmentOperatorShouldBeRecognized()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner(":="));
dynamic assignmentToken = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.ASSIGNMENT_OPERATOR, assignmentToken.getType());
Assert.Equal(":=", assignmentToken.getLexeme());
}
public void readMultipleDifferentSingleCharTokensSeparatedWithWhiteSpace()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner("\\ !\t+\n\n&"));
Assert.Equal(MiniPLTokenType.BACKSLASH, this.tokenScanner.readNextToken().getType());
Assert.Equal(MiniPLTokenType.LOGICAL_NOT, this.tokenScanner.readNextToken().getType());
Assert.Equal(MiniPLTokenType.PLUS, this.tokenScanner.readNextToken().getType());
Assert.Equal(MiniPLTokenType.LOGICAL_AND, this.tokenScanner.readNextToken().getType());
}
public void umlautInIdentifierShouldBeAllowed()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner("ääkköset"));
dynamic token = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.IDENTIFIER, token.getType());
Assert.Equal("ääkköset", token.getLexeme());
}
public void ifNoMoreTokensThenReturnNullToken()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner("!"));
this.tokenScanner.readNextToken();
dynamic nullToken = this.tokenScanner.readNextToken();
Assert.True(nullToken == null);
}
public void readSimpleStringLiterals(String source)
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner(source));
dynamic stringToken = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.STRING_LITERAL, stringToken.getType());
Assert.Equal(source.Substring(1, source.Length - 2), stringToken.getLexeme());
}
public void readIntegerLiterals(String source)
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner(source));
dynamic token = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.INTEGER_LITERAL, token.getType());
Assert.Equal(Convert.ToInt32(source), Convert.ToInt32(token.getLexeme()));
}
public void checkCorrectNumberOfEscapeCharactersInStringLiteral()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner("\"\\\\\\\""));
dynamic stringToken = this.tokenScanner.readNextToken();
Assert.True(stringToken.getLexeme().Length == 3);
Assert.Equal("\\\\\\", stringToken.getLexeme());
}
public void readIdentifierTokenTest(String source)
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner(source));
dynamic token = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.IDENTIFIER, token.getType());
Assert.Equal(source, token.getLexeme());
}
public void sourceCanEndInComment()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner(";\n // This is just a comment"));
dynamic lastToken = this.tokenScanner.readNextToken();
dynamic shouldBeNull = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.SEMICOLON, lastToken.getType());
Assert.True(shouldBeNull == null);
}
public void complexSingleLineCommentTest()
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner(";\n // This is just a comment\n;"));
dynamic firstSemicolon = this.tokenScanner.readNextToken();
dynamic lastSemicolon = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.SEMICOLON, firstSemicolon.getType());
Assert.Equal(MiniPLTokenType.SEMICOLON, lastSemicolon.getType());
}
public void invalidTokenShouldHaveCorrectLexeme(String source)
{
this.tokenScanner = new MiniPLTokenScanner(new Scanner(source));
this.tokenScanner.readNextToken();
dynamic invalidToken = this.tokenScanner.readNextToken();
Assert.Equal(MiniPLTokenType.INVALID_TOKEN, invalidToken.getType());
Assert.Equal("/* This comment never ends\nso this should be the lexeme!", invalidToken.getLexeme());
}