예제 #1
0
 protected virtual FunctionDefinition MaybeParseFunctionDefinition(
     TokenStream tokens,
     Node owner,
     FileScope fileScope,
     AnnotationCollection annotations,
     ModifierCollection modifiers)
 {
     return(null);
 }
예제 #2
0
        private FunctionDefinition ParseFunction(
            TokenStream tokens,
            TopLevelConstruct nullableOwner,
            FileScope fileScope,
            AnnotationCollection annotations)
        {
            bool isStatic =
                nullableOwner != null &&
                nullableOwner is ClassDefinition &&
                tokens.PopIfPresent(this.parser.Keywords.STATIC);

            Token functionToken = tokens.PopExpected(this.parser.Keywords.FUNCTION);

            Token functionNameToken = tokens.Pop();

            this.parser.VerifyIdentifier(functionNameToken);

            FunctionDefinition fd = new FunctionDefinition(functionToken, parser.CurrentLibrary, nullableOwner, isStatic, functionNameToken, annotations, fileScope);

            tokens.PopExpected("(");
            List <Token>      argNames      = new List <Token>();
            List <Expression> defaultValues = new List <Expression>();
            bool optionalArgFound           = false;

            while (!tokens.PopIfPresent(")"))
            {
                if (argNames.Count > 0)
                {
                    tokens.PopExpected(",");
                }

                Token      argName      = tokens.Pop();
                Expression defaultValue = null;
                this.parser.VerifyIdentifier(argName);
                if (tokens.PopIfPresent("="))
                {
                    optionalArgFound = true;
                    defaultValue     = this.parser.ExpressionParser.Parse(tokens, fd);
                }
                else if (optionalArgFound)
                {
                    throw new ParserException(argName, "All optional arguments must come at the end of the argument list.");
                }
                argNames.Add(argName);
                defaultValues.Add(defaultValue);
            }

            IList <Executable> code = ParserContext.ParseBlock(parser, tokens, true, fd);

            fd.ArgNames      = argNames.ToArray();
            fd.DefaultValues = defaultValues.ToArray();
            fd.Code          = code.ToArray();

            return(fd);
        }
예제 #3
0
        protected virtual Namespace ParseNamespace(
            TokenStream tokens,
            Node owner,
            FileScope fileScope,
            AnnotationCollection annotations)
        {
            Token namespaceToken = tokens.PopExpected(this.parser.Keywords.NAMESPACE);
            Token first          = tokens.Pop();

            this.parser.VerifyIdentifier(first);
            List <Token> namespacePieces = new List <Token>()
            {
                first
            };
            string namespaceBuilder = first.Value;

            parser.RegisterNamespace(namespaceBuilder);
            while (tokens.PopIfPresent("."))
            {
                Token nsToken = tokens.Pop();
                this.parser.VerifyIdentifier(nsToken);
                namespacePieces.Add(nsToken);
                namespaceBuilder += "." + nsToken.Value;
                parser.RegisterNamespace(namespaceBuilder);
            }

            string name = string.Join(".", namespacePieces.Select <Token, string>(t => t.Value));

            Namespace namespaceInstance = new Namespace(namespaceToken, name, owner, fileScope, ModifierCollection.EMPTY, annotations);

            tokens.PopExpected("{");
            List <TopLevelEntity> namespaceMembers = new List <TopLevelEntity>();

            while (!tokens.PopIfPresent("}"))
            {
                TopLevelEntity executable = this.Parse(tokens, namespaceInstance, fileScope);
                if (executable is FunctionDefinition ||
                    executable is ClassDefinition ||
                    executable is EnumDefinition ||
                    executable is ConstDefinition ||
                    executable is Namespace)
                {
                    namespaceMembers.Add(executable);
                }
                else
                {
                    throw new ParserException(executable, "Only function, class, and nested namespace declarations may exist as direct members of a namespace.");
                }
            }

            namespaceInstance.Code = namespaceMembers.ToArray();

            return(namespaceInstance);
        }
예제 #4
0
        public void ParseInterpretedCode(string filename, string code)
        {
            FileScope fileScope = new FileScope(filename, this.CurrentScope);
            int       fileId    = this.GetNextFileId();

            this.RegisterFileUsed(filename, code, fileId);
            Token[]     tokenList = Tokenizer.Tokenize(filename, code, fileId, true);
            TokenStream tokens    = new TokenStream(tokenList, filename);

            List <string> namespaceImportsBuilder = new List <string>();

            // Implicitly import the Core library for the current locale.
            LocalizedLibraryView implicitCoreImport = this.LibraryManager.GetCoreLibrary(this);

            namespaceImportsBuilder.Add(implicitCoreImport.Name);
            fileScope.Imports.Add(new ImportStatement(null, implicitCoreImport.Name, this.CurrentLibrary, fileScope));

            while (tokens.HasMore && tokens.IsNext(this.Keywords.IMPORT))
            {
                ImportStatement importStatement = this.ExecutableParser.ParseTopLevel(tokens, null, fileScope) as ImportStatement;
                if (importStatement == null)
                {
                    throw new Exception();
                }
                namespaceImportsBuilder.Add(importStatement.ImportPath);
                LocalizedLibraryView localizedLibraryView = this.LibraryManager.GetOrImportLibrary(this, importStatement.FirstToken, importStatement.ImportPath);
                if (localizedLibraryView == null)
                {
                    this.unresolvedImports.Add(importStatement);
                }
            }

            string[] namespaceImports = namespaceImportsBuilder.ToArray();

            while (tokens.HasMore)
            {
                TopLevelConstruct executable = this.ExecutableParser.ParseTopLevel(tokens, null, fileScope);

                if (executable is ImportStatement)
                {
                    throw this.GenerateParseError(
                              ErrorMessages.ALL_IMPORTS_MUST_OCCUR_AT_BEGINNING_OF_FILE,
                              executable.FirstToken);
                }

                this.CurrentScope.AddExecutable(executable);
            }
        }
예제 #5
0
        protected virtual EnumDefinition ParseEnumDefinition(
            TokenStream tokens,
            Node owner,
            FileScope fileScope,
            ModifierCollection modifiers,
            AnnotationCollection annotations)
        {
            Token enumToken = tokens.PopExpected(this.parser.Keywords.ENUM);
            Token nameToken = tokens.Pop();

            this.parser.VerifyIdentifier(nameToken);
            string         name = nameToken.Value;
            EnumDefinition ed   = new EnumDefinition(enumToken, nameToken, owner, fileScope, modifiers, annotations);

            tokens.PopExpected("{");
            bool              nextForbidden = false;
            List <Token>      items         = new List <Token>();
            List <Expression> values        = new List <Expression>();

            while (!tokens.PopIfPresent("}"))
            {
                if (nextForbidden)
                {
                    tokens.PopExpected("}");                // crash
                }
                Token enumItem = tokens.Pop();
                this.parser.VerifyIdentifier(enumItem);
                if (tokens.PopIfPresent("="))
                {
                    values.Add(this.parser.ExpressionParser.Parse(tokens, ed));
                }
                else
                {
                    values.Add(null);
                }
                nextForbidden = !tokens.PopIfPresent(",");
                items.Add(enumItem);
            }

            ed.SetItems(items, values);
            return(ed);
        }
예제 #6
0
        public void ParseFile(string filename, string code)
        {
            FileScope fileScope = new FileScope(filename, code, this.CurrentScope, this.GetNextFileId());

            this.RegisterFileUsed(fileScope, code);
            TokenStream tokens = new TokenStream(fileScope);

            List <string> namespaceImportsBuilder = new List <string>();

            // Implicitly import the Core library for the current locale.
            LocalizedAssemblyView implicitCoreImport = this.ScopeManager.GetCoreLibrary(this);

            namespaceImportsBuilder.Add(implicitCoreImport.Name);
            fileScope.Imports.Add(new ImportStatement(null, implicitCoreImport.Name, fileScope));

            while (tokens.HasMore &&
                   (tokens.IsNext(this.Keywords.IMPORT) ||
                    (this.IsCSharpCompat && tokens.IsNext("using"))))
            {
                ImportStatement importStatement = this.TopLevelParser.ParseImport(tokens, fileScope);
                if (importStatement == null)
                {
                    throw new Exception();
                }
                namespaceImportsBuilder.Add(importStatement.ImportPath);
                LocalizedAssemblyView localizedAssemblyView = this.ScopeManager.GetOrImportAssembly(this, importStatement.FirstToken, importStatement.ImportPath);
                if (localizedAssemblyView == null)
                {
                    this.unresolvedImports.Add(importStatement);
                }
            }

            string[] namespaceImports = namespaceImportsBuilder.ToArray();

            while (tokens.HasMore)
            {
                this.CurrentScope.AddExecutable(this.TopLevelParser.Parse(tokens, null, fileScope));
            }
        }
예제 #7
0
        internal virtual ImportStatement ParseImport(TokenStream tokens, FileScope fileScope)
        {
            Token importToken = this.parser.IsCSharpCompat
                ? tokens.PopExpected(parser.Keywords.IMPORT, "using")
                : tokens.PopExpected(parser.Keywords.IMPORT);

            List <string> importPathBuilder = new List <string>();

            while (!tokens.PopIfPresent(";"))
            {
                if (importPathBuilder.Count > 0)
                {
                    tokens.PopExpected(".");
                }

                Token pathToken = tokens.Pop();
                parser.VerifyIdentifier(pathToken);
                importPathBuilder.Add(pathToken.Value);
            }
            string importPath = string.Join(".", importPathBuilder);

            return(new ImportStatement(importToken, importPath, fileScope));
        }
예제 #8
0
 protected abstract FunctionDefinition ParseFunction(
     TokenStream tokens,
     TopLevelEntity nullableOwner,
     FileScope fileScope,
     ModifierCollection modifiers,
     AnnotationCollection annotations);
예제 #9
0
 public ParserException(FileScope file, string message)
     : base(file.Name + ": " + message)
 {
     this.file            = file;
     this.OriginalMessage = message;
 }
예제 #10
0
        protected virtual ClassDefinition ParseClassDefinition(
            TokenStream tokens,
            Node owner,
            FileScope fileScope,
            ModifierCollection modifiers,
            AnnotationCollection classAnnotations)
        {
            Token classToken     = tokens.PopExpected(this.parser.Keywords.CLASS);
            Token classNameToken = tokens.Pop();

            if (classNameToken.Type != TokenType.WORD)
            {
                throw new ParserException(classNameToken, "This is not a valid class name.");
            }
            List <Token>  baseClassTokens  = new List <Token>();
            List <string> baseClassStrings = new List <string>();

            if (tokens.PopIfPresent(":"))
            {
                if (baseClassTokens.Count > 0)
                {
                    tokens.PopExpected(",");
                }

                Token  baseClassToken = tokens.Pop();
                string baseClassName  = baseClassToken.Value;

                this.parser.VerifyIdentifier(baseClassToken);
                while (tokens.PopIfPresent("."))
                {
                    Token baseClassTokenNext = tokens.Pop();
                    this.parser.VerifyIdentifier(baseClassTokenNext);
                    baseClassName += "." + baseClassTokenNext.Value;
                }

                baseClassTokens.Add(baseClassToken);
                baseClassStrings.Add(baseClassName);
            }

            ClassDefinition cd = new ClassDefinition(
                classToken,
                classNameToken,
                baseClassTokens,
                baseClassStrings,
                owner,
                fileScope,
                modifiers,
                classAnnotations);

            tokens.PopExpected("{");
            List <FunctionDefinition> methods    = new List <FunctionDefinition>();
            List <FieldDefinition>    fields     = new List <FieldDefinition>();
            List <PropertyDefinition> properties = new List <PropertyDefinition>();

            while (!tokens.PopIfPresent("}"))
            {
                this.ParseClassMember(tokens, fileScope, cd, methods, fields, properties);
            }

            cd.Methods = methods.ToArray();
            cd.Fields  = fields.ToArray();

            if (cd.Constructor == null)
            {
                // This should be empty if there is no base class, or just pass along the base class' args if there is.
                cd.Constructor = new ConstructorDefinition(
                    cd,
                    ModifierCollection.EMPTY,
                    new AnnotationCollection(parser));
                if (cd.BaseClassTokens.Length > 0)
                {
                    cd.Constructor.BaseToken = cd.FirstToken;
                    cd.Constructor.SetBaseArgs(new Expression[0]);
                }
            }

            return(cd);
        }
예제 #11
0
 protected abstract ConstDefinition ParseConst(
     TokenStream tokens,
     Node owner,
     FileScope fileScope,
     ModifierCollection modifiers,
     AnnotationCollection annotations);
예제 #12
0
        public TopLevelConstruct ParseTopLevel(
            TokenStream tokens,
            TopLevelConstruct owner,
            FileScope fileScope)
        {
            AnnotationCollection annotations = annotations = this.parser.AnnotationParser.ParseAnnotations(tokens);

            string value = tokens.PeekValue();

            // The returns are inline, so you'll have to refactor or put the check inside each parse call.
            // Or maybe a try/finally.
            TODO.CheckForUnusedAnnotations();

            Token staticToken = null;
            Token finalToken  = null;

            while (value == this.parser.Keywords.STATIC || value == this.parser.Keywords.FINAL)
            {
                if (value == this.parser.Keywords.STATIC && staticToken == null)
                {
                    staticToken = tokens.Pop();
                    value       = tokens.PeekValue();
                }
                if (value == this.parser.Keywords.FINAL && finalToken == null)
                {
                    finalToken = tokens.Pop();
                    value      = tokens.PeekValue();
                }
            }

            if (staticToken != null || finalToken != null)
            {
                if (value != this.parser.Keywords.CLASS)
                {
                    if (staticToken != null)
                    {
                        throw new ParserException(staticToken, "Only classes, methods, and fields may be marked as static");
                    }
                    else
                    {
                        throw new ParserException(finalToken, "Only classes may be marked as final.");
                    }
                }

                if (staticToken != null && finalToken != null)
                {
                    throw new ParserException(staticToken, "Classes cannot be both static and final.");
                }
            }

            if (value == parser.Keywords.IMPORT)
            {
                Token         importToken       = tokens.PopExpected(parser.Keywords.IMPORT);
                List <string> importPathBuilder = new List <string>();
                while (!tokens.PopIfPresent(";"))
                {
                    if (importPathBuilder.Count > 0)
                    {
                        tokens.PopExpected(".");
                    }

                    Token pathToken = tokens.Pop();
                    parser.VerifyIdentifier(pathToken);
                    importPathBuilder.Add(pathToken.Value);
                }
                string importPath = string.Join(".", importPathBuilder);

                return(new ImportStatement(importToken, importPath, parser.CurrentLibrary, fileScope));
            }

            if (value == this.parser.Keywords.NAMESPACE)
            {
                return(this.ParseNamespace(tokens, owner, fileScope, annotations));
            }
            if (value == this.parser.Keywords.CONST)
            {
                return(this.ParseConst(tokens, owner, fileScope, annotations));
            }
            if (value == this.parser.Keywords.FUNCTION)
            {
                return(this.ParseFunction(tokens, owner, fileScope, annotations));
            }
            if (value == this.parser.Keywords.CLASS)
            {
                return(this.ParseClassDefinition(tokens, owner, staticToken, finalToken, fileScope, annotations));
            }
            if (value == this.parser.Keywords.ENUM)
            {
                return(this.ParseEnumDefinition(tokens, owner, fileScope, annotations));
            }
            if (value == this.parser.Keywords.CONSTRUCTOR)
            {
                return(this.ParseConstructor(tokens, owner, annotations));
            }

            throw new ParserException(tokens.Peek(), "Unrecognized token.");
        }
예제 #13
0
        internal virtual TopLevelEntity Parse(
            TokenStream tokens,
            TopLevelEntity owner,
            FileScope fileScope)
        {
            AnnotationCollection annotations = this.parser.AnnotationParser.ParseAnnotations(tokens);

            ModifierCollection modifiers = ModifierCollection.Parse(tokens);

            string value = tokens.PeekValue();

            if (value == this.parser.Keywords.IMPORT)
            {
                throw this.parser.GenerateParseError(
                          ErrorMessages.ALL_IMPORTS_MUST_OCCUR_AT_BEGINNING_OF_FILE,
                          tokens.Pop());
            }

            // The returns are inline, so you'll have to refactor or put the check inside each parse call.
            // Or maybe a try/finally.
            TODO.CheckForUnusedAnnotations();

            if (value == this.parser.Keywords.NAMESPACE)
            {
                return(this.ParseNamespace(tokens, owner, fileScope, annotations));
            }
            if (value == this.parser.Keywords.CONST)
            {
                return(this.ParseConst(tokens, owner, fileScope, modifiers, annotations));
            }
            if (value == this.parser.Keywords.FUNCTION)
            {
                return(this.ParseFunction(tokens, owner, fileScope, modifiers, annotations));
            }
            if (value == this.parser.Keywords.CLASS)
            {
                return(this.ParseClassDefinition(tokens, owner, fileScope, modifiers, annotations));
            }
            if (value == this.parser.Keywords.ENUM)
            {
                return(this.ParseEnumDefinition(tokens, owner, fileScope, modifiers, annotations));
            }
            if (value == this.parser.Keywords.CONSTRUCTOR && owner is ClassDefinition)
            {
                return(this.ParseConstructor(tokens, (ClassDefinition)owner, modifiers, annotations));
            }

            FunctionDefinition nullableFunctionDef = this.MaybeParseFunctionDefinition(tokens, owner, fileScope, annotations, modifiers);

            if (nullableFunctionDef != null)
            {
                return(nullableFunctionDef);
            }

            Token token = tokens.Peek();

            throw ParserException.ThrowException(
                      this.parser.CurrentLocale,
                      ErrorMessages.UNEXPECTED_TOKEN_NO_SPECIFIC_EXPECTATIONS,
                      token,
                      token.Value);
        }
예제 #14
0
        private ClassDefinition ParseClassDefinition(TokenStream tokens, TopLevelConstruct owner, Token staticToken, Token finalToken, FileScope fileScope, AnnotationCollection classAnnotations)
        {
            Token classToken     = tokens.PopExpected(this.parser.Keywords.CLASS);
            Token classNameToken = tokens.Pop();

            this.parser.VerifyIdentifier(classNameToken);
            List <Token>  baseClassTokens  = new List <Token>();
            List <string> baseClassStrings = new List <string>();

            if (tokens.PopIfPresent(":"))
            {
                if (baseClassTokens.Count > 0)
                {
                    tokens.PopExpected(",");
                }

                Token  baseClassToken = tokens.Pop();
                string baseClassName  = baseClassToken.Value;

                this.parser.VerifyIdentifier(baseClassToken);
                while (tokens.PopIfPresent("."))
                {
                    Token baseClassTokenNext = tokens.Pop();
                    this.parser.VerifyIdentifier(baseClassTokenNext);
                    baseClassName += "." + baseClassTokenNext.Value;
                }

                baseClassTokens.Add(baseClassToken);
                baseClassStrings.Add(baseClassName);
            }

            ClassDefinition cd = new ClassDefinition(
                classToken,
                classNameToken,
                baseClassTokens,
                baseClassStrings,
                owner,
                parser.CurrentLibrary,
                staticToken,
                finalToken,
                fileScope,
                classAnnotations);

            tokens.PopExpected("{");
            List <FunctionDefinition> methods              = new List <FunctionDefinition>();
            List <FieldDeclaration>   fields               = new List <FieldDeclaration>();
            ConstructorDefinition     constructorDef       = null;
            ConstructorDefinition     staticConstructorDef = null;

            while (!tokens.PopIfPresent("}"))
            {
                AnnotationCollection annotations = this.parser.AnnotationParser.ParseAnnotations(tokens);

                if (tokens.IsNext(this.parser.Keywords.FUNCTION) ||
                    tokens.AreNext(this.parser.Keywords.STATIC, this.parser.Keywords.FUNCTION))
                {
                    methods.Add(this.parser.ExecutableParser.ParseFunction(tokens, cd, fileScope, annotations));
                }
                else if (tokens.IsNext(this.parser.Keywords.CONSTRUCTOR))
                {
                    if (constructorDef != null)
                    {
                        throw this.parser.GenerateParseError(
                                  ErrorMessages.CLASS_CANNOT_HAVE_MULTIPLE_CONSTRUCTORS,
                                  tokens.Pop());
                    }

                    constructorDef = this.parser.ExecutableParser.ParseConstructor(tokens, cd, annotations);
                }
                else if (tokens.AreNext(this.parser.Keywords.STATIC, this.parser.Keywords.CONSTRUCTOR))
                {
                    tokens.Pop(); // static token
                    if (staticConstructorDef != null)
                    {
                        throw new ParserException(tokens.Pop(), "Multiple static constructors are not allowed.");
                    }

                    staticConstructorDef = this.parser.ExecutableParser.ParseConstructor(tokens, cd, annotations);
                }
                else if (tokens.IsNext(this.parser.Keywords.FIELD) ||
                         tokens.AreNext(this.parser.Keywords.STATIC, this.parser.Keywords.FIELD))
                {
                    fields.Add(this.parser.ExecutableParser.ParseField(tokens, cd, annotations));
                }
                else
                {
                    tokens.PopExpected("}");
                }

                TODO.CheckForUnusedAnnotations();
            }

            cd.Methods           = methods.ToArray();
            cd.Constructor       = constructorDef;
            cd.StaticConstructor = staticConstructorDef;
            cd.Fields            = fields.ToArray();

            return(cd);
        }
예제 #15
0
        private ConstStatement ParseConst(TokenStream tokens, TopLevelConstruct owner, FileScope fileScope, AnnotationCollection annotations)
        {
            Token          constToken     = tokens.PopExpected(this.parser.Keywords.CONST);
            Token          nameToken      = tokens.Pop();
            ConstStatement constStatement = new ConstStatement(constToken, nameToken, owner, parser.CurrentLibrary, fileScope, annotations);

            this.parser.VerifyIdentifier(nameToken);
            tokens.PopExpected("=");
            constStatement.Expression = this.parser.ExpressionParser.Parse(tokens, constStatement);
            tokens.PopExpected(";");

            return(constStatement);
        }
예제 #16
0
        public TopLevelConstruct ParseTopLevel(
            TokenStream tokens,
            TopLevelConstruct owner,
            FileScope fileScope)
        {
            AnnotationCollection annotations = annotations = this.parser.AnnotationParser.ParseAnnotations(tokens);

            string value = tokens.PeekValue();

            // The returns are inline, so you'll have to refactor or put the check inside each parse call.
            // Or maybe a try/finally.
            TODO.CheckForUnusedAnnotations();

            Token staticToken = null;
            Token finalToken  = null;

            while (value == this.parser.Keywords.STATIC || value == this.parser.Keywords.FINAL)
            {
                if (value == this.parser.Keywords.STATIC && staticToken == null)
                {
                    staticToken = tokens.Pop();
                    value       = tokens.PeekValue();
                }
                if (value == this.parser.Keywords.FINAL && finalToken == null)
                {
                    finalToken = tokens.Pop();
                    value      = tokens.PeekValue();
                }
            }

            if (staticToken != null || finalToken != null)
            {
                if (value != this.parser.Keywords.CLASS)
                {
                    if (staticToken != null)
                    {
                        throw ParserException.ThrowException(this.parser.CurrentLocale, ErrorMessages.ONLY_CLASSES_METHODS_FIELDS_MAY_BE_STATIC, staticToken);
                    }
                    else
                    {
                        throw ParserException.ThrowException(this.parser.CurrentLocale, ErrorMessages.ONLY_CLASSES_MAY_BE_FINAL, finalToken);
                    }
                }

                if (staticToken != null && finalToken != null)
                {
                    throw ParserException.ThrowException(this.parser.CurrentLocale, ErrorMessages.CLASSES_CANNOT_BE_STATIC_AND_FINAL_SIMULTANEOUSLY, staticToken);
                }
            }

            if (value == parser.Keywords.IMPORT)
            {
                Token         importToken       = tokens.PopExpected(parser.Keywords.IMPORT);
                List <string> importPathBuilder = new List <string>();
                while (!tokens.PopIfPresent(";"))
                {
                    if (importPathBuilder.Count > 0)
                    {
                        tokens.PopExpected(".");
                    }

                    Token pathToken = tokens.Pop();
                    parser.VerifyIdentifier(pathToken);
                    importPathBuilder.Add(pathToken.Value);
                }
                string importPath = string.Join(".", importPathBuilder);

                return(new ImportStatement(importToken, importPath, parser.CurrentLibrary, fileScope));
            }

            if (value == this.parser.Keywords.NAMESPACE)
            {
                return(this.ParseNamespace(tokens, owner, fileScope, annotations));
            }
            if (value == this.parser.Keywords.CONST)
            {
                return(this.ParseConst(tokens, owner, fileScope, annotations));
            }
            if (value == this.parser.Keywords.FUNCTION)
            {
                return(this.ParseFunction(tokens, owner, fileScope, annotations));
            }
            if (value == this.parser.Keywords.CLASS)
            {
                return(this.ParseClassDefinition(tokens, owner, staticToken, finalToken, fileScope, annotations));
            }
            if (value == this.parser.Keywords.ENUM)
            {
                return(this.ParseEnumDefinition(tokens, owner, fileScope, annotations));
            }
            if (value == this.parser.Keywords.CONSTRUCTOR)
            {
                return(this.ParseConstructor(tokens, owner, annotations));
            }

            Token token = tokens.Peek();

            throw ParserException.ThrowException(
                      this.parser.CurrentLocale,
                      ErrorMessages.UNEXPECTED_TOKEN_NO_SPECIFIC_EXPECTATIONS,
                      token,
                      token.Value);
        }
 public FileScopedEntityLookup SetFileScope(FileScope fileScope)
 {
     this.fileScope = fileScope;
     return(this);
 }
예제 #18
0
파일: Tokenizer.cs 프로젝트: djlw78/crayon
        public static Token[] Tokenize(FileScope file)
        {
            Localization.Locale locale = file.CompilationScope.Locale;
            string code = file.Content;

            // Add a newline and a dummy character at the end.
            // Set the length equal to the code with the newline but without the null terminator.
            // This makes dereferencing the index + 1 code simpler and all makes the check for the end
            // of word tokens and single-line comments easy.
            code += "\n\0";
            int length = code.Length - 1;

            int[] lineByIndex = new int[code.Length];
            int[] colByIndex  = new int[code.Length];
            char  c;
            int   line = 0;
            int   col  = 0;

            for (int i = 0; i < code.Length; ++i)
            {
                c = code[i];
                lineByIndex[i] = line;
                colByIndex[i]  = col;
                if (c == '\n')
                {
                    ++line;
                    col = -1;
                }

                ++col;
            }

            List <Token> tokens = new List <Token>();

            TokenMode mode        = TokenMode.NORMAL;
            char      modeSubtype = ' ';
            int       tokenStart  = 0;
            string    tokenValue;
            char      c2;
            bool      isTokenEnd  = false;
            bool      stringIsRaw = false;

            for (int i = 0; i < length; ++i)
            {
                c = code[i];

                switch (mode)
                {
                case TokenMode.COMMENT:
                    if (modeSubtype == '*')
                    {
                        if (c == '*' && code[i + 1] == '/')
                        {
                            ++i;
                            mode = TokenMode.NORMAL;
                        }
                    }
                    else
                    {
                        if (c == '\n')
                        {
                            mode = TokenMode.NORMAL;
                        }
                    }
                    break;

                case TokenMode.NORMAL:
                    if (WHITESPACE.Contains(c))
                    {
                        // do nothing
                    }
                    else if (c == '/' && (code[i + 1] == '/' || code[i + 1] == '*'))
                    {
                        mode        = TokenMode.COMMENT;
                        modeSubtype = code[++i];
                    }
                    else if (IsIdentifierChar(c))
                    {
                        tokenStart = i;
                        mode       = TokenMode.WORD;
                    }
                    else if (c == '"' | c == '\'')
                    {
                        tokenStart  = i;
                        mode        = TokenMode.STRING;
                        modeSubtype = c;
                        stringIsRaw = tokens.Count > 0 && tokens[tokens.Count - 1].Value == "@";
                    }
                    else
                    {
                        if (c == '.')
                        {
                            c2 = code[i + 1];
                            if (c2 >= '0' && c2 <= '9')
                            {
                                mode       = TokenMode.WORD;
                                tokenStart = i++;
                            }
                        }

                        if (mode == TokenMode.NORMAL)
                        {
                            tokens.Add(new Token(c.ToString(), TokenType.PUNCTUATION, file, lineByIndex[i], colByIndex[i]));
                        }
                    }
                    break;

                case TokenMode.STRING:
                    if (c == modeSubtype)
                    {
                        tokenValue = code.Substring(tokenStart, i - tokenStart + 1);
                        tokens.Add(new Token(tokenValue, TokenType.STRING, file, lineByIndex[i], colByIndex[i]));
                        mode = TokenMode.NORMAL;
                    }
                    else if (!stringIsRaw && c == '\\')
                    {
                        ++i;
                    }
                    break;

                case TokenMode.WORD:
                    isTokenEnd = false;
                    if (IsIdentifierChar(c))
                    {
                        // do nothing
                    }
                    else if (c == '.')
                    {
                        if (code[tokenStart] >= '0' && code[tokenStart] <= '9')
                        {
                            // do nothing
                        }
                        else
                        {
                            isTokenEnd = true;
                        }
                    }
                    else
                    {
                        isTokenEnd = true;
                    }

                    if (isTokenEnd)
                    {
                        tokenValue = code.Substring(tokenStart, i - tokenStart);
                        c          = tokenValue[0];
                        TokenType type = TokenType.WORD;
                        if ((c >= '0' && c <= '9') || c == '.')
                        {
                            type = TokenType.NUMBER;
                        }
                        else if (!locale.Keywords.IsValidVariable(tokenValue))
                        {
                            type = TokenType.KEYWORD;
                        }
                        tokens.Add(new Token(tokenValue, type, file, lineByIndex[tokenStart], colByIndex[tokenStart]));
                        mode = TokenMode.NORMAL;
                        --i;
                    }
                    break;
                }
            }

            switch (mode)
            {
            case TokenMode.COMMENT:
                throw new ParserException(file, "There is an unclosed comment in this file.");

            case TokenMode.STRING:
                throw new ParserException(file, "There is an unclosed string in this file.");

            case TokenMode.WORD:
                throw new System.InvalidOperationException();

            default:
                break;
            }

            return(tokens.ToArray());
        }
예제 #19
0
 public TokenStream(FileScope file)
 {
     this.file        = file;
     this.innerStream = new InnerTokenStream(Tokenizer.Tokenize(file));
 }
예제 #20
0
 private void RegisterFileUsed(FileScope file, string code)
 {
     this.filesUsed.Add(file.ID, file.Name + "\n" + code);
 }