public bool TheSame(IdentifierToken identifier) { return(identifier.Lexeme == Identifier.Lexeme); }
override internal bool FindNextToken() { int startPosition = _reader.Position; // Dealing with whitespace? if (_reader.SinkMultipleWhiteSpace()) { current = new WhitespaceToken(); return(true); } // Check for one-line comment else if (_reader.Sink("//")) { // Looks like a one-line comment. Follow it to the End-of-line _reader.SinkToEndOfLine(); current = new CommentToken(); return(true); } // Check for multi-line comment else if (_reader.Sink("/*")) { _reader.SinkUntil("*/"); // Was the ending */ found? if (_reader.EndOfLines) { // No. There was a /* without a */. Return this a syntax error token. current = new CSharpTokenizer.EndOfFileInsideCommentToken(); return(true); } current = new CommentToken(); return(true); } // Handle chars else if (_reader.Sink("\'")) { while (_reader.CurrentCharacter != '\'') { if (_reader.Sink("\\")) { /* reader.Skip the escape sequence. * This isn't exactly right. We should detect: * * simple-escape-sequence: one of \' \" \\ \0 \a \b \f \n \r \t \v * * hexadecimal-escape-sequence: * \x hex-digit hex-digit[opt] hex-digit[opt] hex-digit[opt] */ } _reader.SinkCharacter(); } if (_reader.SinkCharacter() != '\'') { Debug.Assert(false, "Code defect in tokenizer: Should have yielded a closing tick."); } current = new CSharpTokenizer.CharLiteralToken(); return(true); } // Check for verbatim string else if (_reader.Sink("@\"")) { do { // Inside a verbatim string "" is treated as a special character while (_reader.Sink("\"\"")) { } }while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"'); // Can't end a file inside a string if (_reader.EndOfLines) { current = new EndOfFileInsideStringToken(); return(true); } // reader.Skip the ending quote. current = new StringLiteralToken(); current.InnerText = _reader.GetCurrentMatchedString(startPosition).Substring(1); return(true); } // Check for a quoted string. else if (_reader.Sink("\"")) { while (_reader.CurrentCharacter == '\\' || _reader.MatchRegularStringLiteral()) { // See if we have an escape sequence. if (_reader.SinkCharacter() == '\\') { // This is probably an escape character. if (_reader.SinkStringEscape()) { // This isn't nearly right. We just do barely enough to make a string // with an embedded escape sequence return _some_ string whose start and // end match the real bounds of the string. } else { // This is a compiler error. _reader.SinkCharacter(); current = new CSharpTokenizer.UnrecognizedStringEscapeToken(); return(true); } } } // Is it a newline? if (TokenChar.IsNewLine(_reader.CurrentCharacter)) { current = new CSharpTokenizer.NewlineInsideStringToken(); return(true); } // Create the token. if (_reader.SinkCharacter() != '\"') { Debug.Assert(false, "Defect in tokenizer: Should have yielded a terminating quote."); } current = new StringLiteralToken(); return(true); } // Identifier or keyword? else if ( // From 2.4.2 Identifiers: A '@' can be used to prefix an identifier so that a keyword can be used as an identifier. _reader.CurrentCharacter == '@' || _reader.MatchNextIdentifierStart() ) { if (_reader.CurrentCharacter == '@') { _reader.SinkCharacter(); } // Now, the next character must be an identifier start. if (!_reader.SinkIdentifierStart()) { current = new ExpectedIdentifierToken(); return(true); } // Sink the rest of the identifier. while (_reader.SinkIdentifierPart()) { } string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition); switch (identifierOrKeyword) { default: if (Array.IndexOf(s_keywordList, identifierOrKeyword) >= 0) { current = new KeywordToken(); return(true); } // If the identifier starts with '@' then we need to strip it off. // The '@' is for escaping so that we can have an identifier called // the same thing as a reserved keyword (i.e. class, if, foreach, etc) string identifier = _reader.GetCurrentMatchedString(startPosition); if (identifier.StartsWith("@", StringComparison.Ordinal)) { identifier = identifier.Substring(1); } // Create the token. current = new IdentifierToken(); current.InnerText = identifier; return(true); case "false": case "true": current = new BooleanLiteralToken(); return(true); case "null": current = new CSharpTokenizer.NullLiteralToken(); return(true); } } // Open scope else if (_reader.Sink("{")) { current = new CSharpTokenizer.OpenScopeToken(); return(true); } // Close scope else if (_reader.Sink("}")) { current = new CSharpTokenizer.CloseScopeToken(); return(true); } // Hexidecimal integer literal else if (_reader.SinkIgnoreCase("0x")) { // Sink the hex digits. if (!_reader.SinkMultipleHexDigits()) { current = new ExpectedValidHexDigitToken(); return(true); } // Skip the L, U, l, u, ul, etc. _reader.SinkLongIntegerSuffix(); current = new HexIntegerLiteralToken(); return(true); } // Decimal integer literal else if (_reader.SinkMultipleDecimalDigits()) { // reader.Skip the L, U, l, u, ul, etc. _reader.SinkLongIntegerSuffix(); current = new DecimalIntegerLiteralToken(); return(true); } // Check for single-digit operators and punctuators else if (_reader.SinkOperatorOrPunctuator()) { current = new OperatorOrPunctuatorToken(); return(true); } // Preprocessor line else if (_reader.CurrentCharacter == '#') { if (_reader.Sink("#if")) { current = new OpenConditionalDirectiveToken(); } else if (_reader.Sink("#endif")) { current = new CloseConditionalDirectiveToken(); } else { current = new PreprocessorToken(); } _reader.SinkToEndOfLine(); return(true); } // We didn't recognize the token, so this is a syntax error. _reader.SinkCharacter(); current = new UnrecognizedToken(); return(true); }
public FbxNode(IdentifierToken identifier) { Identifier = identifier; }
/// <summary> /// Create a new block argument. /// </summary> /// <param name="parent">Block closure that defines the argument node.</param> /// <param name="colon">Colon that precedes the argument name.</param> /// <param name="token">Identifier token containing the name of the argument.</param> protected internal BlockArgumentNode(BlockNode parent, SpecialCharacterToken colon, IdentifierToken token) : base(token) { #if DEBUG if (parent == null) { throw new ArgumentNullException("parent"); } if (colon == null) { throw new ArgumentNullException("colon"); } #endif this.Colon = colon; this.Parent = parent; }
public AtomicNameNode(IdentifierToken identifier) : this(ParseNodeType.Name, identifier) { }
/// <summary> /// member_access = expression operator member_name /// operator = "." /// member_name = identifier /// </summary> /// <param name="expression"></param> /// <param name="name"></param> public ArmMemberAccessExpressionAst(ArmExpressionAst expression, SyntaxToken operatorToken, IdentifierToken name) { this.Expression = expression ?? throw new ArgumentNullException(nameof(expression)); this.OperatorToken = operatorToken ?? throw new ArgumentNullException(nameof(operatorToken)); this.Name = name ?? throw new ArgumentNullException(nameof(name)); }
public static Assignment ParseAssignment(this IEnumerator <Token> tokenStream, IdentifierToken id) { var valueToken = GetNext(tokenStream); switch (valueToken) { case IntegerToken: break; case FloatToken: break; case BooleanToken: break; case StringToken: break; default: throw ParsingException.CreateError(valueToken, "value"); } ExpectNext <SemicolonToken>(tokenStream); return(new Assignment(id, valueToken)); }
public PoolVariableDefinitionNode(IdentifierToken poolName) : base(poolName) { }
public PoolConstantDefinitionNode(IdentifierToken poolName) : base(poolName) { }
public Variable(IdentifierToken token) : base(token) { }
public IdentifierExpression(IdentifierToken idToken) { IdToken = idToken; }
public NamespaceNode(IdentifierToken identifier) => Identifier = identifier;
/// <summary> /// function_reference = function_name /// function_name = identifier /// </summary> /// <param name="name"></param> public ArmFunctionReferenceAst(IdentifierToken name) { this.Name = name ?? throw new ArgumentNullException(nameof(name)); }
public override Node Make(Parser parser) { IdentifierToken id = parser.AssertToken <IdentifierToken>(); return(new Variable(id)); }
private Variable(IdentifierToken id) { this.id = id; region = id.Region; }
public PropertyAccessToken(int position, TokenImpl host, IdentifierToken property) : base(position) { this.Host = host; this.Property = property; }
public void TestTokenProperties() { IdentifierToken result = new IdentifierToken("identifier"); Assert.AreEqual("identifier", result.Value); }
/* * Method: FindNextToken * * Find the next token. Return 'true' if one was found. False, otherwise. */ override internal bool FindNextToken() { int startPosition = _reader.Position; // VB docs claim whitespace is Unicode category Zs. However, // this category does not contain tabs. Assuming a less restrictive // definition for whitespace... if (_reader.SinkWhiteSpace()) { while (_reader.SinkWhiteSpace()) { } // Now, we need to check for the line continuation character. if (_reader.SinkLineContinuationCharacter()) // Line continuation is '_' { // Save the current position because we may need to come back here. int savePosition = _reader.Position - 1; // Skip all whitespace after the '_' while (_reader.SinkWhiteSpace()) { } // Now, skip all the newlines. // Need at least one newline for this to count as line continuation. int count = 0; while (_reader.SinkNewLine()) { ++count; } if (count > 0) { current = new VisualBasicTokenizer.LineContinuationToken(); return(true); } // Otherwise, fall back to plain old whitespace. _reader.Position = savePosition; } current = new WhitespaceToken(); return(true); } // Line terminators are separate from whitespace and are significant. else if (_reader.SinkNewLine()) { // We want one token per line terminator. current = new VisualBasicTokenizer.LineTerminatorToken(); return(true); } // Check for a comment--either those that start with ' or rem. else if (_reader.SinkLineCommentStart()) { // Skip to the first EOL. _reader.SinkToEndOfLine(); current = new CommentToken(); return(true); } // Identifier or keyword? else if ( // VB allows escaping of identifiers by surrounding them with [] // In other words, // Date is a keyword but, // [Date] is an identifier. _reader.CurrentCharacter == '[' || _reader.MatchNextIdentifierStart() ) { bool escapedIdentifier = false; if (_reader.CurrentCharacter == '[') { escapedIdentifier = true; _reader.SinkCharacter(); // Now, the next character must be an identifier start. if (!_reader.SinkIdentifierStart()) { current = new ExpectedIdentifierToken(); return(true); } } // Sink the rest of the identifier. while (_reader.SinkIdentifierPart()) { } // If this was an escaped identifier the we need to get the terminating ']'. if (escapedIdentifier) { if (!_reader.Sink("]")) { current = new ExpectedIdentifierToken(); return(true); } } else { // Escaped identifiers are not allowed to have trailing type character. _reader.SinkTypeCharacter(); // Type character is optional. } // An identifier that is only a '_' is illegal because it is // ambiguous with line continuation string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition); if (identifierOrKeyword == "_" || identifierOrKeyword == "[_]" || identifierOrKeyword == "[]") { current = new ExpectedIdentifierToken(); return(true); } // Make an upper-case version in order to check whether this may be a keyword. string upper = identifierOrKeyword.ToUpper(CultureInfo.InvariantCulture); switch (upper) { default: if (Array.IndexOf(s_keywordList, upper) >= 0) { current = new KeywordToken(); return(true); } // Create the token. current = new IdentifierToken(); // Trim off the [] if this is an escaped identifier. if (escapedIdentifier) { current.InnerText = identifierOrKeyword.Substring(1, identifierOrKeyword.Length - 2); } return(true); case "FALSE": case "TRUE": current = new BooleanLiteralToken(); return(true); } } // Is it a hex integer? else if (_reader.SinkHexIntegerPrefix()) { if (!_reader.SinkMultipleHexDigits()) { current = new ExpectedValidHexDigitToken(); return(true); } // Sink a suffix if there is one. _reader.SinkIntegerSuffix(); current = new HexIntegerLiteralToken(); return(true); } // Is it an octal integer? else if (_reader.SinkOctalIntegerPrefix()) { if (!_reader.SinkMultipleOctalDigits()) { current = new VisualBasicTokenizer.ExpectedValidOctalDigitToken(); return(true); } // Sink a suffix if there is one. _reader.SinkIntegerSuffix(); current = new VisualBasicTokenizer.OctalIntegerLiteralToken(); return(true); } // Is it a decimal integer? else if (_reader.SinkMultipleDecimalDigits()) { // Sink a suffix if there is one. _reader.SinkDecimalIntegerSuffix(); current = new DecimalIntegerLiteralToken(); return(true); } // Preprocessor line else if (_reader.CurrentCharacter == '#') { if (_reader.SinkIgnoreCase("#if")) { current = new OpenConditionalDirectiveToken(); } else if (_reader.SinkIgnoreCase("#end if")) { current = new CloseConditionalDirectiveToken(); } else { current = new PreprocessorToken(); } _reader.SinkToEndOfLine(); return(true); } // Is it a separator? else if (_reader.SinkSeparatorCharacter()) { current = new VisualBasicTokenizer.SeparatorToken(); return(true); } // Is it an operator? else if (_reader.SinkOperator()) { current = new OperatorToken(); return(true); } // A string? else if (_reader.Sink("\"")) { do { // Inside a verbatim string "" is treated as a special character while (_reader.Sink("\"\"")) { } }while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"'); // Can't end a file inside a string if (_reader.EndOfLines) { current = new EndOfFileInsideStringToken(); return(true); } current = new StringLiteralToken(); return(true); } // We didn't recognize the token, so this is a syntax error. _reader.SinkCharacter(); current = new UnrecognizedToken(); return(true); }
public FunctionCall(IdentifierToken name, IList <Ast> args) : base(name) { Name = name; Arguments = args; }
// Constant Binding - cannot be changed /// <summary> /// Create a new argument node. /// </summary> /// <param name="token">Identifier token containing the name of the argument.</param> protected ArgumentNode(IdentifierToken token) { this.Token = token; }
/// <inheritdoc /> public Node Visit(BlockParseNode d) { var parameters = new List <Node>(); Node forcedPattern = null; foreach (ParseNode p in d.Parameters) { var id = p as IdentifierParseNode; var tppn = p as TypedParameterParseNode; var vappn = p as VarArgsParameterParseNode; if (id != null) { parameters.Add(new ParameterNode(id.Token, id)); } else if (tppn != null) { parameters.Add( new ParameterNode(tppn.Token, tppn.Name as IdentifierParseNode, tppn.Type.Visit(this))); } else if (vappn != null) { // Inside could be either an identifier or a // TypedParameterParseNode - check for both. var inIPN = vappn.Name as IdentifierParseNode; var inTPPN = vappn.Name as TypedParameterParseNode; if (inIPN != null) { parameters.Add(new ParameterNode(inIPN.Token, inIPN, true // Variadic )); } else if (inTPPN != null) { parameters.Add(new ParameterNode(inTPPN.Token, inTPPN.Name as IdentifierParseNode, true, // Variadic inTPPN.Type.Visit(this) )); } } else if (p is NumberParseNode || p is StringLiteralParseNode || p is OperatorParseNode) { parameters.Add(p.Visit(this)); } else if (p is ParenthesisedParseNode) { var tok = p.Token; var it = new IdentifierToken(tok.module, tok.line, tok.column, "_"); id = new IdentifierParseNode(it); parameters.Add(new ParameterNode(tok, id, p.Visit(this))); } else { throw new Exception("unimplemented - unusual parameters"); } } var ret = new BlockNode(d.Token, d, parameters, map(d.Body), forcedPattern); return(ret); }
/** * This is the main lexing algorithm. It consumes source file as string and puts out token list. * Token consists of token type and range that token spans. */ public static IToken[] Parse(string s) { LinkedList <IToken> ll = new LinkedList <IToken>(); // use five kinds of token to produce token stream. var index = 0; var next = 0; while (index < s.Length) { // try to parse as number if (NumberToken.ParseNumber(s, index, out var tokenNumber, out next)) { ll.AddLast(tokenNumber); index = next; continue; } // then try to parse as string if (StringToken.ParseString(s, index, out var tokenString, out next)) { ll.AddLast(tokenString); index = next; continue; } // then try to parse as identifier if (IdentifierToken.ParseIdentifier(s, index, out var tokenIdentifier, out next)) { ll.AddLast(tokenIdentifier); index = next; continue; } // then try to parse as comment if (CommentToken.ParseComment(s, index, out var tokensComment, out next)) { foreach (var t in tokensComment) { ll.AddLast(t); } index = next; continue; } // then try to parse as symbol token if (SymbolToken.ParseSymbol(s, index, out var tokenSymbol, out next)) { ll.AddLast(tokenSymbol); index = next; continue; } if (Char.IsWhiteSpace(s[index])) { // skip spaces index++; continue; } // otherwise token is unknown throw new Exception("unknown token " + s[index] + " at position " + index); } // return collected tokens return(ll.ToArray()); }
public EnumElementAst(QualifierListAst qualifierList, IdentifierToken enumElementName, IEnumElementValueAst enumElementValue) { this.QualifierList = qualifierList ?? new QualifierListAst.Builder().Build(); this.EnumElementName = enumElementName; this.EnumElementValue = enumElementValue; }
public GenericNameNode(IdentifierToken name, ParseNodeList typeArguments) : base(ParseNodeType.GenericName, name) { TypeArguments = typeArguments; }
protected AtomicNameNode(ParseNodeType nodeType, IdentifierToken identifier) : base(nodeType, identifier) { Identifier = identifier; }
public IEnumerable <IToken> Tokenize(SafeStreamReader source) { uint lineNumber = 1; uint lexemeStartPositionInLine = 1; uint absolutePosition = 1; var maybeCurrentChar = Option <int> .None; var currentLexemeBuffer = new StringBuilder(); var maybeToken = Option <IToken> .None;; while ((maybeCurrentChar = source.Read()).IsSome) { var currentChar = maybeCurrentChar.Value(); maybeToken = Option <IToken> .None; switch (currentChar) { case var c when string.IsNullOrWhiteSpace(char.ConvertFromUtf32(c)): // if a whitespace was encountered - strip it // and yield whatever in the buffer to the output maybeToken = FlushBuffer( currentLexemeBuffer, ref absolutePosition, lineNumber, ref lexemeStartPositionInLine ); if (maybeToken.IsSome) { yield return(maybeToken.ValueUnsafe()); } switch (c) { case '\r': yield return(source.Read() .Some <IToken>(cn => cn == '\n' ? (IToken) new NewLineSymbolToken( absolutePosition, lineNumber, lexemeStartPositionInLine ) : (IToken) new UnrecognizedToken( $"\r{cn}", absolutePosition, lineNumber, lexemeStartPositionInLine ) ) .None(new UnrecognizedToken( $"\r", absolutePosition, lineNumber, lexemeStartPositionInLine )) ); absolutePosition += 2; lineNumber += 1; lexemeStartPositionInLine = 1; break; case '\n': yield return(new NewLineSymbolToken( absolutePosition, lineNumber, lexemeStartPositionInLine )); absolutePosition += 1; lineNumber += 1; lexemeStartPositionInLine = 1; break; default: absolutePosition += 1; lexemeStartPositionInLine += 1; break; } break; case '.': var currentLexeme = currentLexemeBuffer.ToString(); var maybeBeforeToken = IntegerLiteralToken.FromString( currentLexeme, absolutePosition, lineNumber, lexemeStartPositionInLine ) || IdentifierToken.FromString( currentLexeme, absolutePosition, lineNumber, lexemeStartPositionInLine ) || UnrecognizedToken.FromString( currentLexeme, absolutePosition, lineNumber, lexemeStartPositionInLine ) ; var tokes = source.Peek() .Some <ImmutableList <IToken> >(c => { var result = ImmutableList <IToken> .Empty; IToken tokenToAdd = null; switch (c) { case var _ when IsDigit(char.ConvertFromUtf32(c)): currentLexemeBuffer.Append('.'); return(ImmutableList <IToken> .Empty); case '.': absolutePosition += maybeBeforeToken .Map(t => (uint)t.Lexeme.Length) .IfNone(0); lexemeStartPositionInLine += maybeBeforeToken .Some(t => (uint)t.Lexeme.Length) .None(0u); tokenToAdd = new RangeSymbolToken( absolutePosition, lineNumber, lexemeStartPositionInLine ); result = maybeBeforeToken .ToImmutableList() .Add(tokenToAdd); source.Read(); currentLexemeBuffer.Clear(); lexemeStartPositionInLine += (uint)(tokenToAdd?.Lexeme.Length ?? 0); absolutePosition += (uint)(tokenToAdd?.Lexeme.Length ?? 0); return(result); default: absolutePosition += maybeBeforeToken .Map(t => (uint)t.Lexeme.Length) .IfNone(0); lexemeStartPositionInLine += maybeBeforeToken .Some(t => (uint)t.Lexeme.Length) .None(0u); tokenToAdd = new DotSymbolToken( absolutePosition, lineNumber, lexemeStartPositionInLine ); result = maybeBeforeToken .ToImmutableList() .Add(tokenToAdd); currentLexemeBuffer.Clear(); lexemeStartPositionInLine += (uint)(tokenToAdd?.Lexeme.Length ?? 0); absolutePosition += (uint)(tokenToAdd?.Lexeme.Length ?? 0); return(result); } }) .None(() => { absolutePosition += maybeBeforeToken .Map(t => (uint)t.Lexeme.Length) .IfNone(0); lexemeStartPositionInLine += maybeBeforeToken .Some(t => (uint)t.Lexeme.Length) .None(0u); var tokenToAdd = new DotSymbolToken( absolutePosition, lineNumber, lexemeStartPositionInLine ); var result = maybeBeforeToken .ToImmutableList() .Add(tokenToAdd); currentLexemeBuffer.Clear(); lexemeStartPositionInLine += (uint)(tokenToAdd?.Lexeme.Length ?? 0); absolutePosition += (uint)(tokenToAdd?.Lexeme.Length ?? 0); return(result); }) ; foreach (var token in tokes) { yield return(token); } break; case '/': maybeToken = FlushBuffer( currentLexemeBuffer, ref absolutePosition, lineNumber, ref lexemeStartPositionInLine ); if (maybeToken.IsSome) { yield return(maybeToken.ValueUnsafe()); } yield return(source.Peek() .Some <IToken>(c => { switch (c) { case '/': var commentContent = source.ReadLine(); var commentToken = new CommentToken( $"/{commentContent}", absolutePosition, lineNumber, lexemeStartPositionInLine ); absolutePosition += (uint)commentContent.Length; lineNumber += 1; lexemeStartPositionInLine = 0; return commentToken; case '=': var notEqualsToken = new NotEqualsOperatorToken( absolutePosition, lineNumber, lexemeStartPositionInLine ); source.Read(); absolutePosition += 1; lexemeStartPositionInLine = 1; return notEqualsToken; default: return new DivideOperatorToken( (uint)source.BaseStream.Position, lineNumber, lexemeStartPositionInLine ); } }) .None(() => new DivideOperatorToken( (uint)source.BaseStream.Position, lineNumber, lexemeStartPositionInLine ))); absolutePosition += 1; lexemeStartPositionInLine += 1; break; case ':': maybeToken = FlushBuffer( currentLexemeBuffer, ref absolutePosition, lineNumber, ref lexemeStartPositionInLine ); if (maybeToken.IsSome) { yield return(maybeToken.ValueUnsafe()); } yield return(source.Peek() .Filter(c => c == '=') .Some <IToken>(c => { var result = new AssignmentOperatorToken( absolutePosition, lineNumber, lexemeStartPositionInLine ); source.Read(); absolutePosition += 1; lexemeStartPositionInLine += 1; return result; }) .None(new ColonSymbolToken( absolutePosition, lineNumber, lexemeStartPositionInLine ))); absolutePosition += 1; lexemeStartPositionInLine += 1; break; case '>': maybeToken = FlushBuffer( currentLexemeBuffer, ref absolutePosition, lineNumber, ref lexemeStartPositionInLine ); if (maybeToken.IsSome) { yield return(maybeToken.ValueUnsafe()); } yield return(source.Peek() .Filter(c => c == '=') .Some <IToken>(_ => { var result = new GeOperatorToken( absolutePosition, lineNumber, lexemeStartPositionInLine ); source.Read(); absolutePosition += 1; lexemeStartPositionInLine += 1; return result; }) .None(new GtOperatorToken( (uint)absolutePosition, lineNumber, lexemeStartPositionInLine ))); absolutePosition += 1; lexemeStartPositionInLine += 1; break; case '<': maybeToken = FlushBuffer( currentLexemeBuffer, ref absolutePosition, lineNumber, ref lexemeStartPositionInLine ); if (maybeToken.IsSome) { yield return(maybeToken.ValueUnsafe()); } yield return(source.Peek() .Filter(c => c == '=') .Some <IToken>(_ => { var result = new LeOperatorToken( absolutePosition, lineNumber, lexemeStartPositionInLine ); source.Read(); absolutePosition += 1; lexemeStartPositionInLine += 1; return result; }) .None(new LtOperatorToken( absolutePosition, lineNumber, lexemeStartPositionInLine ))); absolutePosition += 1; lexemeStartPositionInLine += 1; break; case '*': case '%': case '+': case '-': case '=': case ',': case '[': case ']': case '(': case ')': case ';': maybeToken = FlushBuffer( currentLexemeBuffer, ref absolutePosition, lineNumber, ref lexemeStartPositionInLine ); if (maybeToken.IsSome) { yield return(maybeToken.ValueUnsafe()); } yield return(SymbolLexemes .TryGetValue(((char)currentChar).ToString()) .Some(cons => cons( absolutePosition, lineNumber, lexemeStartPositionInLine )) .None(() => new UnrecognizedToken( currentChar.ToString(), absolutePosition, lineNumber, lexemeStartPositionInLine ) )); absolutePosition += 1; lexemeStartPositionInLine += 1; break; default: currentLexemeBuffer.Append(char.ConvertFromUtf32(currentChar)); break; } } maybeToken = FlushBuffer( currentLexemeBuffer, ref absolutePosition, lineNumber, ref lexemeStartPositionInLine ); if (maybeToken.IsSome) { yield return(maybeToken.ValueUnsafe()); } }
private IExpression ParseUnaryExpression(ref IToken token) { IExpression unaryExpression = ParseLiteralExpression(ref token); if (unaryExpression != null) { token = tokenParser.GetNextToken(); } else if (token is MinusToken) { token = tokenParser.GetNextToken(); return(new UnaryMinusExpression(ParseUnaryExpression(ref token))); } else if (token is IdentifierToken) { IdentifierToken idToken = token as IdentifierToken; token = tokenParser.GetNextToken(); if (token is OpenParenthesisToken) { try { ParameterExpression[] args = ParseArgumentList(ref token); unaryExpression = new FunctionExpression(idToken, args); } catch (SyntaxErrorException e) { throw new SyntaxErrorException(e.Message, e, e.Expression, new FunctionExpression(idToken, e.Args)); } } else if (token is LambdaInvokeToken) { try { token = tokenParser.GetNextToken(); IExpression rightExpression = ParseExpression(ref token); unaryExpression = new LambdaExpression(idToken, rightExpression); } catch (SyntaxErrorException e) { throw new SyntaxErrorException(e.Message, e, e.Expression, new LambdaExpression(idToken, e.RootExpression)); } } else { unaryExpression = new IdentifierExpression(idToken); } } else if (token is OpenParenthesisToken) { token = tokenParser.GetNextToken(); IExpression complexExpression = ParseLogicalOrExpression(ref token); if (!(token is CloseParenthesisToken)) { throw new SyntaxErrorException("')' experado."); } token = tokenParser.GetNextToken(); unaryExpression = complexExpression; } else { throw new SyntaxErrorException("expressão esperada."); } //decide se é uma propery access expression while (token is DotToken) { token = tokenParser.GetNextToken(); if (!(token is IdentifierToken)) { token = tokenParser.GetNextToken(); throw new SyntaxErrorException("identificador esperado", unaryExpression, unaryExpression); } IdentifierToken id = token as IdentifierToken; unaryExpression = new PropertyAccessExpression(unaryExpression, id.IdentifierName); token = tokenParser.GetNextToken(); } return(unaryExpression); }
public AssociationDeclarationAst(QualifierListAst qualifierList, IdentifierToken associationName, IdentifierToken superAssociation, ReadOnlyCollection <IClassFeatureAst> classFeatures) { this.QualifierList = qualifierList ?? new QualifierListAst.Builder().Build(); this.AssociationName = associationName ?? throw new ArgumentNullException(nameof(associationName)); this.SuperAssociation = superAssociation; this.ClassFeatures = classFeatures ?? new ReadOnlyCollection <IClassFeatureAst>(new List <IClassFeatureAst>()); }
internal static TableQualifier Create(IdentifierToken identifier, List <MemberAccessToken> memberAccessors) { return(new TableQualifier(identifier.Content, memberAccessors.Select(accessor => accessor.MemberName))); }
private static TriggerTemplate ParseTriggerTemplate(IEnumerator <Token> tokenStream, IdentifierToken id) { var oldNum = tokenStream .ExpectNext <IntegerToken>() .ValueAsUshort(token => ParsingException.CreateError(token, "UShort value")); tokenStream.ExpectNext <OpenBraceToken>(); var block = tokenStream.ParseBlock(id); return(ReadTriggerTemplate(oldNum, block)); }
/* * Method: FindNextToken * * Find the next token. Return 'true' if one was found. False, otherwise. */ internal override bool FindNextToken() { int startPosition = _reader.Position; // VB docs claim whitespace is Unicode category Zs. However, // this category does not contain tabs. Assuming a less restrictive // definition for whitespace... if (_reader.SinkWhiteSpace()) { while (_reader.SinkWhiteSpace()) { } // Now, we need to check for the line continuation character. if (_reader.SinkLineContinuationCharacter()) // Line continuation is '_' { // Save the current position because we may need to come back here. int savePosition = _reader.Position - 1; // Skip all whitespace after the '_' while (_reader.SinkWhiteSpace()) { } // Now, skip all the newlines. // Need at least one newline for this to count as line continuation. int count = 0; while (_reader.SinkNewLine()) { ++count; } if (count > 0) { current = new VisualBasicTokenizer.LineContinuationToken(); return true; } // Otherwise, fall back to plain old whitespace. _reader.Position = savePosition; } current = new WhitespaceToken(); return true; } // Line terminators are separate from whitespace and are significant. else if (_reader.SinkNewLine()) { // We want one token per line terminator. current = new VisualBasicTokenizer.LineTerminatorToken(); return true; } // Check for a comment--either those that start with ' or rem. else if (_reader.SinkLineCommentStart()) { // Skip to the first EOL. _reader.SinkToEndOfLine(); current = new CommentToken(); return true; } // Identifier or keyword? else if ( // VB allows escaping of identifiers by surrounding them with [] // In other words, // Date is a keyword but, // [Date] is an identifier. _reader.CurrentCharacter == '[' || _reader.MatchNextIdentifierStart() ) { bool escapedIdentifier = false; if (_reader.CurrentCharacter == '[') { escapedIdentifier = true; _reader.SinkCharacter(); // Now, the next character must be an identifier start. if (!_reader.SinkIdentifierStart()) { current = new ExpectedIdentifierToken(); return true; } } // Sink the rest of the identifier. while (_reader.SinkIdentifierPart()) { } // If this was an escaped identifier the we need to get the terminating ']'. if (escapedIdentifier) { if (!_reader.Sink("]")) { current = new ExpectedIdentifierToken(); return true; } } else { // Escaped identifiers are not allowed to have trailing type character. _reader.SinkTypeCharacter(); // Type character is optional. } // An identifier that is only a '_' is illegal because it is // ambiguous with line continuation string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition); if (identifierOrKeyword == "_" || identifierOrKeyword == "[_]" || identifierOrKeyword == "[]") { current = new ExpectedIdentifierToken(); return true; } // Make an upper-case version in order to check whether this may be a keyword. string upper = identifierOrKeyword.ToUpper(CultureInfo.InvariantCulture); switch (upper) { default: if (Array.IndexOf(s_keywordList, upper) >= 0) { current = new KeywordToken(); return true; } // Create the token. current = new IdentifierToken(); // Trim off the [] if this is an escaped identifier. if (escapedIdentifier) { current.InnerText = identifierOrKeyword.Substring(1, identifierOrKeyword.Length - 2); } return true; case "FALSE": case "TRUE": current = new BooleanLiteralToken(); return true; } } // Is it a hex integer? else if (_reader.SinkHexIntegerPrefix()) { if (!_reader.SinkMultipleHexDigits()) { current = new ExpectedValidHexDigitToken(); return true; } // Sink a suffix if there is one. _reader.SinkIntegerSuffix(); current = new HexIntegerLiteralToken(); return true; } // Is it an octal integer? else if (_reader.SinkOctalIntegerPrefix()) { if (!_reader.SinkMultipleOctalDigits()) { current = new VisualBasicTokenizer.ExpectedValidOctalDigitToken(); return true; } // Sink a suffix if there is one. _reader.SinkIntegerSuffix(); current = new VisualBasicTokenizer.OctalIntegerLiteralToken(); return true; } // Is it a decimal integer? else if (_reader.SinkMultipleDecimalDigits()) { // Sink a suffix if there is one. _reader.SinkDecimalIntegerSuffix(); current = new DecimalIntegerLiteralToken(); return true; } // Preprocessor line else if (_reader.CurrentCharacter == '#') { if (_reader.SinkIgnoreCase("#if")) { current = new OpenConditionalDirectiveToken(); } else if (_reader.SinkIgnoreCase("#end if")) { current = new CloseConditionalDirectiveToken(); } else { current = new PreprocessorToken(); } _reader.SinkToEndOfLine(); return true; } // Is it a separator? else if (_reader.SinkSeparatorCharacter()) { current = new VisualBasicTokenizer.SeparatorToken(); return true; } // Is it an operator? else if (_reader.SinkOperator()) { current = new OperatorToken(); return true; } // A string? else if (_reader.Sink("\"")) { do { // Inside a verbatim string "" is treated as a special character while (_reader.Sink("\"\"")) { } } while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"'); // Can't end a file inside a string if (_reader.EndOfLines) { current = new EndOfFileInsideStringToken(); return true; } current = new StringLiteralToken(); return true; } // We didn't recognize the token, so this is a syntax error. _reader.SinkCharacter(); current = new UnrecognizedToken(); return true; }
/* * Method: FindNextToken * * Find the next token. Return 'true' if one was found. False, otherwise. */ override internal bool FindNextToken() { int startPosition = _reader.Position; // Dealing with whitespace? if (_reader.SinkMultipleWhiteSpace()) { current = new WhitespaceToken(); return true; } // Check for one-line comment else if (_reader.Sink("//")) { // Looks like a one-line comment. Follow it to the End-of-line _reader.SinkToEndOfLine(); current = new CommentToken(); return true; } // Check for multi-line comment else if (_reader.Sink("/*")) { _reader.SinkUntil("*/"); // Was the ending */ found? if (_reader.EndOfLines) { // No. There was a /* without a */. Return this a syntax error token. current = new CSharpTokenizer.EndOfFileInsideCommentToken(); return true; } current = new CommentToken(); return true; } // Handle chars else if (_reader.Sink("\'")) { while (_reader.CurrentCharacter != '\'') { if (_reader.Sink("\\")) { /* reader.Skip the escape sequence. This isn't exactly right. We should detect: simple-escape-sequence: one of \' \" \\ \0 \a \b \f \n \r \t \v hexadecimal-escape-sequence: \x hex-digit hex-digit[opt] hex-digit[opt] hex-digit[opt] */ } _reader.SinkCharacter(); } if (_reader.SinkCharacter() != '\'') { Debug.Assert(false, "Code defect in tokenizer: Should have yielded a closing tick."); } current = new CSharpTokenizer.CharLiteralToken(); return true; } // Check for verbatim string else if (_reader.Sink("@\"")) { do { // Inside a verbatim string "" is treated as a special character while (_reader.Sink("\"\"")) { } } while (!_reader.EndOfLines && _reader.SinkCharacter() != '\"'); // Can't end a file inside a string if (_reader.EndOfLines) { current = new EndOfFileInsideStringToken(); return true; } // reader.Skip the ending quote. current = new StringLiteralToken(); current.InnerText = _reader.GetCurrentMatchedString(startPosition).Substring(1); return true; } // Check for a quoted string. else if (_reader.Sink("\"")) { while (_reader.CurrentCharacter == '\\' || _reader.MatchRegularStringLiteral()) { // See if we have an escape sequence. if (_reader.SinkCharacter() == '\\') { // This is probably an escape character. if (_reader.SinkStringEscape()) { // This isn't nearly right. We just do barely enough to make a string // with an embedded escape sequence return _some_ string whose start and // end match the real bounds of the string. } else { // This is a compiler error. _reader.SinkCharacter(); current = new CSharpTokenizer.UnrecognizedStringEscapeToken(); return true; } } } // Is it a newline? if (TokenChar.IsNewLine(_reader.CurrentCharacter)) { current = new CSharpTokenizer.NewlineInsideStringToken(); return true; } // Create the token. if (_reader.SinkCharacter() != '\"') { Debug.Assert(false, "Defect in tokenizer: Should have yielded a terminating quote."); } current = new StringLiteralToken(); return true; } // Identifier or keyword? else if ( // From 2.4.2 Identifiers: A '@' can be used to prefix an identifier so that a keyword can be used as an identifier. _reader.CurrentCharacter == '@' || _reader.MatchNextIdentifierStart() ) { if (_reader.CurrentCharacter == '@') { _reader.SinkCharacter(); } // Now, the next character must be an identifier start. if (!_reader.SinkIdentifierStart()) { current = new ExpectedIdentifierToken(); return true; } // Sink the rest of the identifier. while (_reader.SinkIdentifierPart()) { } string identifierOrKeyword = _reader.GetCurrentMatchedString(startPosition); switch (identifierOrKeyword) { default: if (Array.IndexOf(s_keywordList, identifierOrKeyword) >= 0) { current = new KeywordToken(); return true; } // If the identifier starts with '@' then we need to strip it off. // The '@' is for escaping so that we can have an identifier called // the same thing as a reserved keyword (i.e. class, if, foreach, etc) string identifier = _reader.GetCurrentMatchedString(startPosition); if (identifier.StartsWith("@", StringComparison.Ordinal)) { identifier = identifier.Substring(1); } // Create the token. current = new IdentifierToken(); current.InnerText = identifier; return true; case "false": case "true": current = new BooleanLiteralToken(); return true; case "null": current = new CSharpTokenizer.NullLiteralToken(); return true; } } // Open scope else if (_reader.Sink("{")) { current = new CSharpTokenizer.OpenScopeToken(); return true; } // Close scope else if (_reader.Sink("}")) { current = new CSharpTokenizer.CloseScopeToken(); return true; } // Hexidecimal integer literal else if (_reader.SinkIgnoreCase("0x")) { // Sink the hex digits. if (!_reader.SinkMultipleHexDigits()) { current = new ExpectedValidHexDigitToken(); return true; } // Skip the L, U, l, u, ul, etc. _reader.SinkLongIntegerSuffix(); current = new HexIntegerLiteralToken(); return true; } // Decimal integer literal else if (_reader.SinkMultipleDecimalDigits()) { // reader.Skip the L, U, l, u, ul, etc. _reader.SinkLongIntegerSuffix(); current = new DecimalIntegerLiteralToken(); return true; } // Check for single-digit operators and punctuators else if (_reader.SinkOperatorOrPunctuator()) { current = new OperatorOrPunctuatorToken(); return true; } // Preprocessor line else if (_reader.CurrentCharacter == '#') { if (_reader.Sink("#if")) { current = new OpenConditionalDirectiveToken(); } else if (_reader.Sink("#endif")) { current = new CloseConditionalDirectiveToken(); } else { current = new PreprocessorToken(); } _reader.SinkToEndOfLine(); return true; } // We didn't recognize the token, so this is a syntax error. _reader.SinkCharacter(); current = new UnrecognizedToken(); return true; }