Ejemplo n.º 1
0
        private static void AddSeparator(SourceCode file, ICollection <BasicToken> tokens, SourcePosition position)
        {
            switch (file.Current)
            {
            case '-':
                switch (file.Next)
                {
                case '>':
                    tokens.Add(new BasicToken(TokenType.PickChild, position));
                    break;

                case '=':
                    tokens.Add(new BasicToken(TokenType.MinusEqual, position));
                    break;

                default:
                    tokens.Add(new BasicToken(TokenType.Minus, position));
                    break;
                }
                break;

            case '+':
                switch (file.Next)
                {
                case '=':
                    tokens.Add(new BasicToken(TokenType.AddEqual, position));
                    break;

                default:
                    tokens.Add(new BasicToken(TokenType.Add, position));
                    break;
                }
                break;

            case '*':
                switch (file.Next)
                {
                case '=':
                    tokens.Add(new BasicToken(TokenType.MultiplyEqual, position));
                    break;

                default:
                    tokens.Add(new BasicToken(TokenType.Multiply, position));
                    break;
                }
                break;

            case '/':
                switch (file.Next)
                {
                case '=':
                    tokens.Add(new BasicToken(TokenType.DivideEqual, position));
                    break;

                default:
                    tokens.Add(new BasicToken(TokenType.Divide, position));
                    break;
                }
                break;

            case '>':
                switch (file.Next)
                {
                case '=':
                    tokens.Add(new BasicToken(TokenType.GreaterEqual, position));
                    break;

                default:
                    tokens.Add(new BasicToken(TokenType.Greater, position));
                    break;
                }
                break;

            case '<':
                switch (file.Next)
                {
                case '=':
                    tokens.Add(new BasicToken(TokenType.LesserEqual, position));
                    break;

                default:
                    tokens.Add(new BasicToken(TokenType.Lesser, position));
                    break;
                }
                break;

            case '[':
                tokens.Add(new BasicToken(TokenType.PluginCallStart, position));
                break;

            case ']':
                tokens.Add(new BasicToken(TokenType.PluginCallEnd, position));
                break;

            case '!':
                switch (file.Next)
                {
                case '=':
                    tokens.Add(new BasicToken(TokenType.LogicNotEqual, position));
                    break;

                default:
                    tokens.Add(new BasicToken(TokenType.LogicNot, position));
                    break;
                }
                break;

            case '@':
                switch (file.Next)
                {
                case '#':
                    tokens.Add(new BasicToken(TokenType.Constant, position));
                    break;

                default:
                    tokens.Add(new BasicToken(TokenType.Variable, position));
                    break;
                }
                break;

            case '=':
                switch (file.Next)
                {
                case '=':
                    tokens.Add(new BasicToken(TokenType.LogicEqual, position));
                    break;

                default:
                    tokens.Add(new BasicToken(TokenType.Equal, position));
                    break;
                }
                break;

            case '(':
                tokens.Add(new BasicToken(TokenType.LeftParenthesis, position));
                break;

            case ')':
                tokens.Add(new BasicToken(TokenType.RightParenthesis, position));
                break;
            }
        }
Ejemplo n.º 2
0
        /// <summary>
        /// 执行词法分析
        /// </summary>
        /// <param name="source">源文件内容</param>
        /// <param name="identifier">源文件ID</param>
        /// <returns></returns>
        /// <exception cref="CompileException"></exception>
        public static IEnumerable <BasicToken> Lex(string source, CodeIdentifier identifier)
        {
            var file     = new SourceCode(source);
            var position = new SourcePosition();
            var tokens   = new List <BasicToken>();
            var indent   = new SourceIndent();

            while (file.HasNext)
            {
                if (file.Current == '/' && file.Next == '/')   // 跳过所有注释
                {
                    file.MoveToNextLineBreak();
                    continue;
                }
                if (file.Current == '\n')   // 缩进处理及跳过空行、空逻辑行
                {
                    position = position.NextLine();
                    if (file.Next == '\n')   // 空行直接跳过
                    {
                        file.MoveToNext();
                        continue;
                    }
                    if (file.Next != ' ' && file.Next != '/' && file[2] != '/') // 如果行不以空格开头,尝试取消所有缩进
                    {
                        if (file.Next == '/' && file[2] == '/')                 // 如果行仅包含注释则不予处理
                        {
                            file.MoveToNextLineBreak();
                            continue;
                        }
                        tokens.Add(new BasicToken(TokenType.LineBreak, position));
                        // ReSharper disable once AccessToModifiedClosure
                        tokens.AddRange(indent.ShrinkTo(0).Select(i => new BasicToken(TokenType.LeaveScope, position)));
                    }
                    else     // 否则计算该行缩进值
                    {
                        file.MoveToNext();
                        var offset = file.IndexOfUntilNot(' ');
                        if (file[offset] == '\n' || file[offset] == '/' && file[offset + 1] == '/')   // 空行和只有注释的行不予任何处理
                        {
                            file.MoveToNextLineBreak();
                            continue;
                        }
                        tokens.Add(new BasicToken(TokenType.LineBreak, position));
                        if (offset < indent.Length)   // 缩进小于当前缩进则取消部分缩进
                        {
                            var approachingIndent = indent.FindApproachingIndent(offset);
                            if (approachingIndent != offset)   // 当缩进比最大缩进小时,不接受未出现过的缩进值
                            {
                                throw new CompileException(identifier, position, $"Unable to create indent: Indent length {offset} is not acceptable, try {approachingIndent}");
                            }
                            // ReSharper disable once AccessToModifiedClosure
                            tokens.AddRange(indent.ShrinkTo(approachingIndent).Select(i => new BasicToken(TokenType.LeaveScope, position)));
                        }
                        else if (offset > indent.Length)     // 缩进大于当前缩进则增加缩进
                        {
                            indent.Push(offset - indent.Length);
                            tokens.Add(new BasicToken(TokenType.CreateScope, position));
                        }
                        file.Move(offset - 1);
                        position = position.MoveColumn(offset);
                    }
                    file.MoveToNext(); // 换行符处理结束后移动游标一位以对应CodePosition

                    // 如果去掉空白后行以#开头则表示带角色描述的快速对话。快速对话中不能使用可编程语法,也不能中途换行,直接解析至行尾即可。
                    if (file.Current == '#')
                    {
                        file.MoveToNext();
                        position = position.NextColumn();
                        var nextSpace = file.IndexOf(' ');
                        var lineBreak = file.IndexOf('\n');
                        if (nextSpace < 0 || nextSpace > lineBreak)   // 对话内容不能为空
                        {
                            throw new CompileException(identifier, position, "Unable to create quick dialogue: dialogue starts with character must has content");
                        }
                        if (file.Current == ' ')   // 角色描述不能为空
                        {
                            throw new CompileException(identifier, position, "Unable to create quick dialogue: dialogue starts with character must has character definition");
                        }
                        tokens.Add(new StringToken(TokenType.DialogueSpeaker, position, file.CopyContent(nextSpace).ExecuteEscapeCharacters(), false));
                        position = position.MoveColumn(nextSpace);
                        tokens.Add(new StringToken(TokenType.DialogueContent, position, file.CopyContent(nextSpace + 1, lineBreak).ExecuteEscapeCharacters(), false));
                        file.Move(lineBreak);
                        position = position.MoveColumn(lineBreak - nextSpace);
                        continue;
                    }
                }

                if (file.Current == '\0')
                {
                    break;
                }

                if (file.Current == '\'')   // 不可翻译字符串常量
                {
                    file.MoveToNext();
                    var stringEnd = file.IndexOfWithEscapeRecognize('\'');
                    if (stringEnd < 0)
                    {
                        throw new CompileException(identifier, position, "String constant has no end mark");
                    }
                    var result = file.CopyContent(stringEnd);
                    tokens.Add(new StringToken(TokenType.String, position, result.ExecuteEscapeCharacters(), false));
                    position = result.Contains('\n')
                        ? position.NextLine().MoveLine(result.Count(e => e == '\n') - 1).MoveColumn(result.Length - result.LastIndexOf('\n') - 1)
                        : position.MoveColumn(stringEnd + 2);
                    file.Move(stringEnd + 1);
                    continue;
                }

                if (file.Current == '"')   // 可翻译字符串常量
                {
                    file.MoveToNext();
                    var stringEnd = file.IndexOfWithEscapeRecognize('"');
                    if (stringEnd < 0)
                    {
                        throw new CompileException(identifier, position, "Translatable string constant has no end mark");
                    }
                    var result = file.CopyContent(stringEnd);
                    tokens.Add(new StringToken(TokenType.String, position, result.ExecuteEscapeCharacters(), true));
                    position = result.Contains('\n')
                        ? position.NextLine().MoveLine(result.Count(e => e == '\n') - 1).MoveColumn(result.Length - result.LastIndexOf('\n') - 1)
                        : position.MoveColumn(stringEnd + 2);
                    file.Move(stringEnd + 1);
                    continue;
                }

                if (file.Current >= '0' && file.Current <= '9')   // 数字常量
                {
                    var numberEnd = file.IndexOfUntilNot('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F', 'x', 'X', '.');
                    var number    = file.CopyContent(numberEnd);
                    CreateNumberToken(identifier, tokens, position, number, false);
                    file.Move(numberEnd);
                    position = position.MoveColumn(numberEnd - file.Offset);
                    continue;
                }

                if (file.Current == '-' && file.Next >= '0' && file.Next <= '9')   // 负数字常量
                {
                    file.MoveToNext();
                    var numberEnd = file.IndexOfUntilNot('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F', 'x', 'X', '.');
                    var number    = file.CopyContent(numberEnd);
                    CreateNumberToken(identifier, tokens, position, number, true);
                    file.Move(numberEnd);
                    position = position.MoveColumn(numberEnd - file.Offset + 1);
                    continue;
                }

                if (file.Current == ';' && file.Previous != '\\')   // 指令
                {
                    file.MoveToNext();
                    position = position.NextColumn();
                    var index = file.IndexOf(' ', '\n');
                    if (file.StartsWith(Keywords.Function))
                    {
                        tokens.Add(new BasicToken(TokenType.Function, position));
                    }
                    else if (file.StartsWith(Keywords.If))
                    {
                        tokens.Add(new BasicToken(TokenType.If, position));
                    }
                    else if (file.StartsWith(Keywords.ElseIf))
                    {
                        tokens.Add(new BasicToken(TokenType.ElseIf, position));
                    }
                    else if (file.StartsWith(Keywords.Else))
                    {
                        tokens.Add(new BasicToken(TokenType.Else, position));
                    }
                    else if (file.StartsWith(Keywords.WhileLoop + ' '))
                    {
                        tokens.Add(new BasicToken(TokenType.Loop, position));
                    }
                    else if (file.StartsWith(Keywords.Return))
                    {
                        tokens.Add(new BasicToken(TokenType.Return, position));
                    }
                    else if (file.StartsWith(Keywords.Call))
                    {
                        tokens.Add(new BasicToken(TokenType.FunctionCall, position));
                    }
                    else if (file.StartsWith(Keywords.Import))
                    {
                        tokens.Add(new BasicToken(TokenType.Import, position));
                    }
                    else if (file.StartsWith(Keywords.Export))
                    {
                        tokens.Add(new BasicToken(TokenType.Export, position));
                    }
                    else
                    {
                        throw new CompileException(identifier, position, $"Unknown keyword {file.CopyContent(index)}");
                    }
                    file.Move(index);
                    position = position.MoveColumn(index);
                    continue;
                }

                if (file.StartsWith(Keywords.Separators))   // 操作符和分隔符
                {
                    AddSeparator(file, tokens, position);
                    if (file.Current == '-' && (file.Next == '>' || file.Next == '=') ||
                        file.Current == '@' && file.Next == '#' ||
                        file.Current == '+' && file.Next == '=' ||
                        file.Current == '*' && file.Next == '=' ||
                        file.Current == '>' && file.Next == '=' ||
                        file.Current == '<' && file.Next == '=' ||
                        file.Current == '=' && file.Next == '=' ||
                        file.Current == '!' && file.Next == '=' ||
                        file.Current == '/' && file.Next == '=')
                    {
                        file.MoveToNext();
                        position = position.NextColumn();
                    }
                    file.MoveToNext();
                    position = position.NextColumn();
                    var nextSeparator = file.IndexOfWithEscapeRecognize(Keywords.Separators); // 有可能是换行,如果有问题待到语法分析时再报错,这里因为没有足够的状态记录,无法检测是否合法
                    if (nextSeparator < 0)
                    {
                        break;
                    }
                    if (file[nextSeparator] == '-' && file[nextSeparator + 1] >= '0' && file[nextSeparator + 1] <= '9')   // 负数
                    {
                        var initialEndPosition = nextSeparator;
                        file.Move(initialEndPosition + 1);
                        nextSeparator = file.IndexOfWithEscapeRecognize(Keywords.Separators) + 1;
                        file.Move(-initialEndPosition - 1);
                    }
                    var content = file.CopyContent(nextSeparator).Trim();
                    if (content.Length > 1 && content[0] == '-' && content[1] >= '0' && content[1] <= '9')
                    {
                        CreateNumberToken(identifier, tokens, position, content.Substring(1), true);
                    }
                    else if (content.Length > 0)
                    {
                        if (content[0] >= '0' && content[0] <= '9')
                        {
                            CreateNumberToken(identifier, tokens, position, content, false);
                        }
                        else
                        {
                            tokens.Add(new StringToken(TokenType.String, position, content, false));
                        }
                    }
                    file.Move(nextSeparator);
                    position = position.MoveColumn(nextSeparator);
                    continue;
                }

                if (file.Current == ' ')   // 空格
                {
                    file.MoveToNext();
                    position = position.NextColumn();
                    continue;
                }

                // 如果所有条件都不满足,那么一定是普通的无角色对话
                var nextLine = file.IndexOf('\n');
                tokens.Add(new StringToken(TokenType.DialogueContent, position, file.CopyContent(nextLine).ExecuteEscapeCharacters(), false));
                tokens.Add(new BasicToken(TokenType.LineBreak, position));
                file.Move(nextLine);
            }
            file.Reset(); // 重置游标至-1
            if (tokens.Count == 0 || tokens.Last().Type != TokenType.LineBreak)
            {
                tokens.Add(new BasicToken(TokenType.LineBreak, position));
            }
            if (indent.Length <= 0)
            {
                return(tokens);
            }
            tokens.AddRange(indent.ShrinkTo(0).Select(i => new BasicToken(TokenType.LeaveScope, position)));
            tokens.Add(new BasicToken(TokenType.LineBreak, position));
            return(tokens);
        }