public TokenWithSpan(TokenWithSpan currentToken, JSToken newToken) { _indexResolver = currentToken._indexResolver; _start = currentToken._start; _end = currentToken._end; _token = newToken; }
private static SourceSpan GetSpan(TokenWithSpan context) { return(new SourceSpan( new SourceLocation( context.Start, context.StartLine, context.StartColumn ), new SourceLocation( context.End, context.EndLine, context.EndColumn ) )); }
protected bool AddArgument(TokenWithSpan token, out string errMsg) { errMsg = null; string key = token.Token.Value.ToString(); if (!_internalArguments.ContainsKey(key)) { _internalArguments.Add(key, token); _arguments.Add(token, null); _argsInOrder.Add(token); return(true); } errMsg = string.Format("Duplicate argument found: {0}", key); return(false); }
private static List <TokenWithSpan> ScanTokens(string code, bool collectWarnings, params ErrorInfo[] errors) { CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings); var scanner = new JSScanner(code, errorSink, new CodeSettings() { AllowShebangLine = true }); List <TokenWithSpan> tokens = new List <TokenWithSpan>(); for (TokenWithSpan curToken = scanner.ScanNextTokenWithSpan(true); curToken.Token != JSToken.EndOfFile; curToken = scanner.ScanNextTokenWithSpan(true)) { tokens.Add(curToken); } errorSink.CheckErrors(errors); return(tokens); }
private IEnumerable <ITagSpan <IOutliningRegionTag> > ProcessSuite(NormalizedSnapshotSpanCollection spans, GeneroAst ast, ModuleNode moduleNode, ITextSnapshot snapshot, bool isTopLevel) { if (moduleNode != null) { TokenWithSpan[] regionTokens = new TokenWithSpan[0]; object val; if (ast.TryGetAttribute(moduleNode, NodeAttributes.CodeRegions, out val)) { regionTokens = val as TokenWithSpan[]; } var outlinables = new List <IOutlinableResult>(); GetOutlinableResults(moduleNode, ref outlinables); if (_taggerProvider._CustomCommentOutliningProvider != null) { TokenWithSpan[] commentLines = new TokenWithSpan[0]; if (ast.TryGetAttribute(moduleNode, NodeAttributes.NonCodeRegionComments, out val)) { commentLines = val as TokenWithSpan[]; outlinables.AddRange(_taggerProvider._CustomCommentOutliningProvider.GetRegions(commentLines)); } } foreach (var child in outlinables.Union(GetRegions(regionTokens))) { SnapshotSpan?span = ShouldInclude(child, spans); if (span == null) { continue; } TagSpan tagSpan = GetOutlineSpan(snapshot, child); if (tagSpan != null) { yield return(tagSpan); } } } }
public new static bool TryParseNode(IParser parser, out RecordDefinitionNode defNode, bool isPublic = false) { defNode = null; bool result = false; if (parser.PeekToken(TokenKind.RecordKeyword)) { result = true; defNode = new RecordDefinitionNode(); defNode.StartIndex = parser.Token.Span.Start; defNode.Location = parser.TokenLocation; defNode._isPublic = isPublic; parser.NextToken(); // move past the record keyword if (parser.PeekToken(TokenKind.LikeKeyword)) { // get db info parser.NextToken(); if (!parser.PeekToken(TokenCategory.Identifier) && parser.PeekToken(TokenKind.Colon, 2)) { parser.NextToken(); // advance to the database name defNode.MimicDatabaseName = parser.Token.Token.Value.ToString(); parser.NextToken(); // advance to the colon } FglNameExpression tableName; if (FglNameExpression.TryParseNode(parser, out tableName)) { defNode.MimicTableName = tableName; if (!tableName.Name.EndsWith(".*")) { parser.ReportSyntaxError("A mimicking record must reference a table as follows: \"[tablename].*\"."); } } else { parser.ReportSyntaxError("Invalid database table name found in record definition."); } } else { AttributeSpecifier attribSpec; if (AttributeSpecifier.TryParseNode(parser, out attribSpec)) { defNode.Attribute = attribSpec; } bool advance = true; TokenWithSpan tok = default(TokenWithSpan); while (parser.PeekToken(TokenCategory.Identifier) || parser.PeekToken(TokenCategory.Keyword)) { //if (advance) //{ parser.NextToken(); tok = parser.Token; //} //else //{ // // reset // tok = parser.Token.Token; // advance = true; //} TypeReference tr; if (TypeReference.TryParseNode(parser, out tr, true)) { if (!defNode.MemberDictionary.ContainsKey(tok.Token.Value.ToString())) { defNode.MemberDictionary.Add(tok.Token.Value.ToString(), new VariableDef(tok.Token.Value.ToString(), tr, tok.Span.Start, true, defNode.Location?.FilePath)); } else { parser.ReportSyntaxError(string.Format("Record field {0} defined more than once.", tok.Token.Value.ToString()), Severity.Error); } } AttributeSpecifier.TryParseNode(parser, out attribSpec); if (parser.MaybeEat(TokenKind.Comma)) { //advance = false; continue; } else if (parser.MaybeEat(TokenKind.EndKeyword)) { if (!parser.MaybeEat(TokenKind.RecordKeyword)) { parser.ReportSyntaxError("Invalid end token in record definition"); } else { defNode.EndIndex = parser.Token.Span.End; defNode.IsComplete = true; } break; } else { parser.ReportSyntaxError("Invalid token within record definition"); break; } } } } return(result); }
private static TokenInfo ToTokenKind(TokenWithSpan context) { switch (context.Token) { case JSToken.Semicolon: // ; return(new TokenInfo( GetSpan(context), TokenCategory.Delimiter, TokenTriggers.None )); case JSToken.RightCurly: // } case JSToken.LeftCurly: // { return(new TokenInfo( GetSpan(context), TokenCategory.Grouping, TokenTriggers.MatchBraces )); case JSToken.Debugger: case JSToken.Var: case JSToken.If: case JSToken.For: case JSToken.Do: case JSToken.While: case JSToken.Continue: case JSToken.Break: case JSToken.Return: case JSToken.With: case JSToken.Switch: case JSToken.Throw: case JSToken.Try: case JSToken.Function: case JSToken.Else: case JSToken.Null: case JSToken.True: case JSToken.False: case JSToken.This: case JSToken.Void: case JSToken.TypeOf: case JSToken.Delete: case JSToken.Case: case JSToken.Catch: case JSToken.Default: case JSToken.Finally: case JSToken.New: case JSToken.Class: case JSToken.Const: case JSToken.Enum: case JSToken.Export: case JSToken.Extends: case JSToken.Import: case JSToken.Super: // ECMA strict reserved words case JSToken.Implements: case JSToken.In: case JSToken.InstanceOf: case JSToken.Interface: case JSToken.Let: case JSToken.Package: case JSToken.Private: case JSToken.Protected: case JSToken.Public: case JSToken.Static: case JSToken.Yield: // always okay for identifiers case JSToken.Get: case JSToken.Set: return(new TokenInfo( GetSpan(context), TokenCategory.Keyword, TokenTriggers.None )); // used by both statement and expression switches // main expression switch case JSToken.Identifier: return(new TokenInfo( GetSpan(context), TokenCategory.Identifier, TokenTriggers.None )); case JSToken.StringLiteral: return(new TokenInfo( GetSpan(context), TokenCategory.StringLiteral, TokenTriggers.None )); case JSToken.IntegerLiteral: case JSToken.NumericLiteral: return(new TokenInfo( GetSpan(context), TokenCategory.NumericLiteral, TokenTriggers.None )); case JSToken.LeftParenthesis: // ( return(new TokenInfo( GetSpan(context), TokenCategory.Grouping, TokenTriggers.ParameterStart | TokenTriggers.MatchBraces )); case JSToken.RightParenthesis: // ) return(new TokenInfo( GetSpan(context), TokenCategory.Grouping, TokenTriggers.ParameterEnd | TokenTriggers.MatchBraces )); case JSToken.LeftBracket: // [ return(new TokenInfo( GetSpan(context), TokenCategory.Grouping, TokenTriggers.MatchBraces )); case JSToken.RightBracket: // ] return(new TokenInfo( GetSpan(context), TokenCategory.Grouping, TokenTriggers.MatchBraces )); case JSToken.AccessField: // . return(new TokenInfo( GetSpan(context), TokenCategory.Operator, TokenTriggers.MemberSelect )); // unary ops case JSToken.Increment: // ++ case JSToken.Decrement: // -- case JSToken.LogicalNot: // ! case JSToken.BitwiseNot: // ~ case JSToken.Plus: // + case JSToken.Minus: // - case JSToken.Multiply: // * case JSToken.Divide: // / case JSToken.Modulo: // % case JSToken.BitwiseAnd: // & case JSToken.BitwiseOr: // | case JSToken.BitwiseXor: // ^ case JSToken.LeftShift: // << case JSToken.RightShift: // >> case JSToken.UnsignedRightShift: // >>> case JSToken.Equal: // == case JSToken.NotEqual: // != case JSToken.StrictEqual: // === case JSToken.StrictNotEqual: // !== case JSToken.LessThan: // < case JSToken.LessThanEqual: // <= case JSToken.GreaterThan: // > case JSToken.GreaterThanEqual: // >= case JSToken.LogicalAnd: // && case JSToken.LogicalOr: // || case JSToken.Assign: // = case JSToken.PlusAssign: // += case JSToken.MinusAssign: // -= case JSToken.MultiplyAssign: // *= case JSToken.DivideAssign: // /= case JSToken.ModuloAssign: // %= case JSToken.BitwiseAndAssign: // &= case JSToken.BitwiseOrAssign: // |= case JSToken.BitwiseXorAssign: // ^= case JSToken.LeftShiftAssign: // <<= case JSToken.RightShiftAssign: // >>= case JSToken.UnsignedRightShiftAssign: // >>>= case JSToken.ConditionalIf: // ? // MUST FOLLOW LastBinaryOp case JSToken.Colon: // : return(new TokenInfo( GetSpan(context), TokenCategory.Operator, TokenTriggers.None )); case JSToken.Comma: // : return(new TokenInfo( GetSpan(context), TokenCategory.Delimiter, TokenTriggers.ParameterNext )); case JSToken.SingleLineComment: // for authoring return(new TokenInfo( GetSpan(context), TokenCategory.LineComment, TokenTriggers.None )); case JSToken.MultipleLineComment: // for authoring case JSToken.UnterminatedComment: // for authoring return(new TokenInfo( GetSpan(context), TokenCategory.Comment, TokenTriggers.None )); case JSToken.RegularExpression: // only returned if the RawTokens flag is set on the scanner return(new TokenInfo( GetSpan(context), TokenCategory.RegularExpressionLiteral, TokenTriggers.None )); } return(new TokenInfo( GetSpan(context), TokenCategory.None, TokenTriggers.None )); }