Ejemplo n.º 1
0
        public Error CreateError(string s)
        {
            var error = new SyntaxError(s);

            error.AddStackTrace(GetLocation(_index));
            return(error);
        }
Ejemplo n.º 2
0
        public static IList <Token> Lex(string code, int fileId)
        {
            var tokens = new List <Token>();

            var index        = 0;
            var buffer       = "";
            var inString     = false;
            var stringStart  = '\0';
            var escaping     = false;
            var inOperator   = false;
            var inNumber     = false;
            var inComment    = false;
            var commentStart = "";

            var line      = 1;
            var character = 1;

            Error createError(string message)
            {
                var error = new SyntaxError(message);

                error.AddStackTrace(new Debug.Location(fileId, line));
                return(error);
            }

            void clearBuffer(Token.Group type, bool force = false)
            {
                if (buffer.Length > 0 || force)
                {
                    if (type == Token.Group.Operator && !Tokens.IsValidOperator(buffer))
                    {
                        for (var start = 0; start < buffer.Length;)
                        {
                            var found = false;

                            for (var end = buffer.Length; end > start; end--)
                            {
                                var sub = buffer.Substring(start, end - start);
                                if (Tokens.IsValidOperator(sub))
                                {
                                    tokens.Add(new Token(sub, type, line, character));
                                    start = end;
                                    found = true;
                                    break;
                                }
                            }

                            if (!found)
                            {
                                throw createError($"Invalid operator '{buffer}'");
                            }
                        }
                    }
                    else
                    {
                        tokens.Add(new Token(buffer, type, line, character));
                    }

                    buffer = "";
                }
            }

            void incrementPosition(char c)
            {
                if (c == '\n')
                {
                    line++;
                    character = 1;
                }
                else
                {
                    character++;
                }
            }

            while (index < code.Length)
            {
                var c = code[index];
                incrementPosition(c);

                if (inOperator)
                {
                    if (IsOperator(c))
                    {
                        buffer += c;
                        index++;
                        continue;
                    }
                    else
                    {
                        inOperator = false;
                        clearBuffer(Token.Group.Operator);
                    }
                }

                if (inNumber)
                {
                    if ((c >= '0' && c <= '9') || c == '.' || c == 'e' || c == 'E' || c == '-')
                    {
                        buffer += c;
                        index++;
                        continue;
                    }
                    else
                    {
                        inNumber = false;
                        clearBuffer(Token.Group.Number);
                    }
                }

                if (inComment)
                {
                    if (c == '\n' && commentStart == "//")
                    {
                        clearBuffer(Token.Group.Comment);
                        inComment = false;
                    }
                    else if (commentStart == "/*" && c == '*' && index + 1 < code.Length && code[index + 1] == '/')
                    {
                        buffer += "*/";
                        clearBuffer(Token.Group.Comment);
                        index    += 2;
                        inComment = false;
                        continue;
                    }
                    else
                    {
                        buffer += c;
                        index++;
                        continue;
                    }
                }

                if (inString)
                {
                    if (escaping)
                    {
                        if (c == '\'' || c == '"' || c == '\\')
                        {
                            buffer += c;
                        }
                        else if (c == 'n')
                        {
                            buffer += '\n';
                        }
                        else if (c == 'b')
                        {
                            buffer += '\b';
                        }
                        else if (c == 'r')
                        {
                            buffer += '\r';
                        }
                        else if (c == 'f')
                        {
                            buffer += '\f';
                        }
                        else if (c == 't')
                        {
                            buffer += '\t';
                        }
                        else if (c == 'v')
                        {
                            buffer += '\v';
                        }
                        else
                        {
                            throw createError("Invalid escaped character '" + c + "'");
                        }

                        escaping = false;
                    }
                    else
                    {
                        if (c == stringStart)
                        {
                            inString = false;
                            clearBuffer(stringStart == '`' ? Token.Group.Template : Token.Group.String, true);
                        }
                        else if (c == '\\')
                        {
                            escaping = true;
                        }
                        else if (c == '\r')
                        {
                            // TODO: ignore this?
                        }
                        else if (c == '\n' && stringStart != '`')
                        {
                            throw createError("Newline in string literal");
                        }
                        else
                        {
                            buffer += c;
                        }
                    }
                }
                else
                {
                    if (IsWhitespace(c))
                    {
                        clearBuffer(Token.Group.Text);
                        if (c != '\r')
                        {
                            tokens.Add(new Token(c.ToString(), Token.Group.WhiteSpace, line, character));
                        }
                    }
                    else if (IsStructure(c))
                    {
                        clearBuffer(Token.Group.Text);
                        tokens.Add(new Token(c.ToString(), Token.Group.Structure, line, character));
                    }
                    else
                    {
                        var htmlIndex = IsHtml(code, index);
                        if (htmlIndex != -1)
                        {
                            clearBuffer(Token.Group.Text);

                            var html = code.Substring(index, htmlIndex - index + 1);
                            tokens.Add(new Token(html, Token.Group.Html, line, character));

                            for (var h = 1; h < html.Length; h++)
                            {
                                incrementPosition(html[h]);
                            }

                            index = htmlIndex;
                        }
                        else if (c == '/' && index + 1 < code.Length && (code[index + 1] == '/' || code[index + 1] == '*'))
                        {
                            clearBuffer(Token.Group.Text);

                            inComment    = true;
                            commentStart = c.ToString() + code[index + 1];
                            buffer       = commentStart;
                            index++;
                        }
                        else if (IsOperator(c))
                        {
                            clearBuffer(Token.Group.Text);

                            if (c == '.')
                            {
                                var next = index + 1 < code.Length ? code[index + 1] : '\0';
                                if (next >= '0' && next <= '9')
                                {
                                    inNumber = true;
                                }
                            }

                            if (!inNumber)
                            {
                                inOperator = true;
                            }

                            buffer += c;
                        }
                        else if (IsString(c))
                        {
                            inString    = true;
                            stringStart = c;
                        }
                        else if (c >= '0' && c <= '9' && buffer.Length == 0)
                        {
                            inNumber = true;
                            buffer  += c;
                        }
                        else
                        {
                            buffer += c;
                        }
                    }
                }

                index++;
            }

            if (inOperator)
            {
                clearBuffer(Token.Group.Operator);
            }
            else if (inNumber)
            {
                clearBuffer(Token.Group.Number);
            }
            else if (inString)
            {
                throw new SyntaxError("Unterminated string literal");
            }
            else
            {
                clearBuffer(Token.Group.Text);
            }

            return(tokens);
        }