private static void AddSeparator(SourceCode file, ICollection <BasicToken> tokens, SourcePosition position)
{
switch (file.Current)
{
case '-':
switch (file.Next)
{
case '>':
tokens.Add(new BasicToken(TokenType.PickChild, position));
break;
case '=':
tokens.Add(new BasicToken(TokenType.MinusEqual, position));
break;
default:
tokens.Add(new BasicToken(TokenType.Minus, position));
break;
}
break;
case '+':
switch (file.Next)
{
case '=':
tokens.Add(new BasicToken(TokenType.AddEqual, position));
break;
default:
tokens.Add(new BasicToken(TokenType.Add, position));
break;
}
break;
case '*':
switch (file.Next)
{
case '=':
tokens.Add(new BasicToken(TokenType.MultiplyEqual, position));
break;
default:
tokens.Add(new BasicToken(TokenType.Multiply, position));
break;
}
break;
case '/':
switch (file.Next)
{
case '=':
tokens.Add(new BasicToken(TokenType.DivideEqual, position));
break;
default:
tokens.Add(new BasicToken(TokenType.Divide, position));
break;
}
break;
case '>':
switch (file.Next)
{
case '=':
tokens.Add(new BasicToken(TokenType.GreaterEqual, position));
break;
default:
tokens.Add(new BasicToken(TokenType.Greater, position));
break;
}
break;
case '<':
switch (file.Next)
{
case '=':
tokens.Add(new BasicToken(TokenType.LesserEqual, position));
break;
default:
tokens.Add(new BasicToken(TokenType.Lesser, position));
break;
}
break;
case '[':
tokens.Add(new BasicToken(TokenType.PluginCallStart, position));
break;
case ']':
tokens.Add(new BasicToken(TokenType.PluginCallEnd, position));
break;
case '!':
switch (file.Next)
{
case '=':
tokens.Add(new BasicToken(TokenType.LogicNotEqual, position));
break;
default:
tokens.Add(new BasicToken(TokenType.LogicNot, position));
break;
}
break;
case '@':
switch (file.Next)
{
case '#':
tokens.Add(new BasicToken(TokenType.Constant, position));
break;
default:
tokens.Add(new BasicToken(TokenType.Variable, position));
break;
}
break;
case '=':
switch (file.Next)
{
case '=':
tokens.Add(new BasicToken(TokenType.LogicEqual, position));
break;
default:
tokens.Add(new BasicToken(TokenType.Equal, position));
break;
}
break;
case '(':
tokens.Add(new BasicToken(TokenType.LeftParenthesis, position));
break;
case ')':
tokens.Add(new BasicToken(TokenType.RightParenthesis, position));
break;
}
}
/// <summary>
/// 执行词法分析
/// </summary>
/// <param name="source">源文件内容</param>
/// <param name="identifier">源文件ID</param>
/// <returns></returns>
/// <exception cref="CompileException"></exception>
public static IEnumerable <BasicToken> Lex(string source, CodeIdentifier identifier)
{
var file = new SourceCode(source);
var position = new SourcePosition();
var tokens = new List <BasicToken>();
var indent = new SourceIndent();
while (file.HasNext)
{
if (file.Current == '/' && file.Next == '/') // 跳过所有注释
{
file.MoveToNextLineBreak();
continue;
}
if (file.Current == '\n') // 缩进处理及跳过空行、空逻辑行
{
position = position.NextLine();
if (file.Next == '\n') // 空行直接跳过
{
file.MoveToNext();
continue;
}
if (file.Next != ' ' && file.Next != '/' && file[2] != '/') // 如果行不以空格开头，尝试取消所有缩进
{
if (file.Next == '/' && file[2] == '/') // 如果行仅包含注释则不予处理
{
file.MoveToNextLineBreak();
continue;
}
tokens.Add(new BasicToken(TokenType.LineBreak, position));
// ReSharper disable once AccessToModifiedClosure
tokens.AddRange(indent.ShrinkTo(0).Select(i => new BasicToken(TokenType.LeaveScope, position)));
}
else // 否则计算该行缩进值
{
file.MoveToNext();
var offset = file.IndexOfUntilNot(' ');
if (file[offset] == '\n' || file[offset] == '/' && file[offset + 1] == '/') // 空行和只有注释的行不予任何处理
{
file.MoveToNextLineBreak();
continue;
}
tokens.Add(new BasicToken(TokenType.LineBreak, position));
if (offset < indent.Length) // 缩进小于当前缩进则取消部分缩进
{
var approachingIndent = indent.FindApproachingIndent(offset);
if (approachingIndent != offset) // 当缩进比最大缩进小时，不接受未出现过的缩进值
{
throw new CompileException(identifier, position, $"Unable to create indent: Indent length {offset} is not acceptable, try {approachingIndent}");
}
// ReSharper disable once AccessToModifiedClosure
tokens.AddRange(indent.ShrinkTo(approachingIndent).Select(i => new BasicToken(TokenType.LeaveScope, position)));
}
else if (offset > indent.Length) // 缩进大于当前缩进则增加缩进
{
indent.Push(offset - indent.Length);
tokens.Add(new BasicToken(TokenType.CreateScope, position));
}
file.Move(offset - 1);
position = position.MoveColumn(offset);
}
file.MoveToNext(); // 换行符处理结束后移动游标一位以对应CodePosition
// 如果去掉空白后行以#开头则表示带角色描述的快速对话。快速对话中不能使用可编程语法，也不能中途换行，直接解析至行尾即可。
if (file.Current == '#')
{
file.MoveToNext();
position = position.NextColumn();
var nextSpace = file.IndexOf(' ');
var lineBreak = file.IndexOf('\n');
if (nextSpace < 0 || nextSpace > lineBreak) // 对话内容不能为空
{
throw new CompileException(identifier, position, "Unable to create quick dialogue: dialogue starts with character must has content");
}
if (file.Current == ' ') // 角色描述不能为空
{
throw new CompileException(identifier, position, "Unable to create quick dialogue: dialogue starts with character must has character definition");
}
tokens.Add(new StringToken(TokenType.DialogueSpeaker, position, file.CopyContent(nextSpace).ExecuteEscapeCharacters(), false));
position = position.MoveColumn(nextSpace);
tokens.Add(new StringToken(TokenType.DialogueContent, position, file.CopyContent(nextSpace + 1, lineBreak).ExecuteEscapeCharacters(), false));
file.Move(lineBreak);
position = position.MoveColumn(lineBreak - nextSpace);
continue;
}
}
if (file.Current == '\0')
{
break;
}
if (file.Current == '\'') // 不可翻译字符串常量
{
file.MoveToNext();
var stringEnd = file.IndexOfWithEscapeRecognize('\'');
if (stringEnd < 0)
{
throw new CompileException(identifier, position, "String constant has no end mark");
}
var result = file.CopyContent(stringEnd);
tokens.Add(new StringToken(TokenType.String, position, result.ExecuteEscapeCharacters(), false));
position = result.Contains('\n')
? position.NextLine().MoveLine(result.Count(e => e == '\n') - 1).MoveColumn(result.Length - result.LastIndexOf('\n') - 1)
: position.MoveColumn(stringEnd + 2);
file.Move(stringEnd + 1);
continue;
}
if (file.Current == '"') // 可翻译字符串常量
{
file.MoveToNext();
var stringEnd = file.IndexOfWithEscapeRecognize('"');
if (stringEnd < 0)
{
throw new CompileException(identifier, position, "Translatable string constant has no end mark");
}
var result = file.CopyContent(stringEnd);
tokens.Add(new StringToken(TokenType.String, position, result.ExecuteEscapeCharacters(), true));
position = result.Contains('\n')
? position.NextLine().MoveLine(result.Count(e => e == '\n') - 1).MoveColumn(result.Length - result.LastIndexOf('\n') - 1)
: position.MoveColumn(stringEnd + 2);
file.Move(stringEnd + 1);
continue;
}
if (file.Current >= '0' && file.Current <= '9') // 数字常量
{
var numberEnd = file.IndexOfUntilNot('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F', 'x', 'X', '.');
var number = file.CopyContent(numberEnd);
CreateNumberToken(identifier, tokens, position, number, false);
file.Move(numberEnd);
position = position.MoveColumn(numberEnd - file.Offset);
continue;
}
if (file.Current == '-' && file.Next >= '0' && file.Next <= '9') // 负数字常量
{
file.MoveToNext();
var numberEnd = file.IndexOfUntilNot('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F', 'x', 'X', '.');
var number = file.CopyContent(numberEnd);
CreateNumberToken(identifier, tokens, position, number, true);
file.Move(numberEnd);
position = position.MoveColumn(numberEnd - file.Offset + 1);
continue;
}
if (file.Current == ';' && file.Previous != '\\') // 指令
{
file.MoveToNext();
position = position.NextColumn();
var index = file.IndexOf(' ', '\n');
if (file.StartsWith(Keywords.Function))
{
tokens.Add(new BasicToken(TokenType.Function, position));
}
else if (file.StartsWith(Keywords.If))
{
tokens.Add(new BasicToken(TokenType.If, position));
}
else if (file.StartsWith(Keywords.ElseIf))
{
tokens.Add(new BasicToken(TokenType.ElseIf, position));
}
else if (file.StartsWith(Keywords.Else))
{
tokens.Add(new BasicToken(TokenType.Else, position));
}
else if (file.StartsWith(Keywords.WhileLoop + ' '))
{
tokens.Add(new BasicToken(TokenType.Loop, position));
}
else if (file.StartsWith(Keywords.Return))
{
tokens.Add(new BasicToken(TokenType.Return, position));
}
else if (file.StartsWith(Keywords.Call))
{
tokens.Add(new BasicToken(TokenType.FunctionCall, position));
}
else if (file.StartsWith(Keywords.Import))
{
tokens.Add(new BasicToken(TokenType.Import, position));
}
else if (file.StartsWith(Keywords.Export))
{
tokens.Add(new BasicToken(TokenType.Export, position));
}
else
{
throw new CompileException(identifier, position, $"Unknown keyword {file.CopyContent(index)}");
}
file.Move(index);
position = position.MoveColumn(index);
continue;
}
if (file.StartsWith(Keywords.Separators)) // 操作符和分隔符
{
AddSeparator(file, tokens, position);
if (file.Current == '-' && (file.Next == '>' || file.Next == '=') ||
file.Current == '@' && file.Next == '#' ||
file.Current == '+' && file.Next == '=' ||
file.Current == '*' && file.Next == '=' ||
file.Current == '>' && file.Next == '=' ||
file.Current == '<' && file.Next == '=' ||
file.Current == '=' && file.Next == '=' ||
file.Current == '!' && file.Next == '=' ||
file.Current == '/' && file.Next == '=')
{
file.MoveToNext();
position = position.NextColumn();
}
file.MoveToNext();
position = position.NextColumn();
var nextSeparator = file.IndexOfWithEscapeRecognize(Keywords.Separators); // 有可能是换行，如果有问题待到语法分析时再报错，这里因为没有足够的状态记录，无法检测是否合法
if (nextSeparator < 0)
{
break;
}
if (file[nextSeparator] == '-' && file[nextSeparator + 1] >= '0' && file[nextSeparator + 1] <= '9') // 负数
{
var initialEndPosition = nextSeparator;
file.Move(initialEndPosition + 1);
nextSeparator = file.IndexOfWithEscapeRecognize(Keywords.Separators) + 1;
file.Move(-initialEndPosition - 1);
}
var content = file.CopyContent(nextSeparator).Trim();
if (content.Length > 1 && content[0] == '-' && content[1] >= '0' && content[1] <= '9')
{
CreateNumberToken(identifier, tokens, position, content.Substring(1), true);
}
else if (content.Length > 0)
{
if (content[0] >= '0' && content[0] <= '9')
{
CreateNumberToken(identifier, tokens, position, content, false);
}
else
{
tokens.Add(new StringToken(TokenType.String, position, content, false));
}
}
file.Move(nextSeparator);
position = position.MoveColumn(nextSeparator);
continue;
}
if (file.Current == ' ') // 空格
{
file.MoveToNext();
position = position.NextColumn();
continue;
}
// 如果所有条件都不满足，那么一定是普通的无角色对话
var nextLine = file.IndexOf('\n');
tokens.Add(new StringToken(TokenType.DialogueContent, position, file.CopyContent(nextLine).ExecuteEscapeCharacters(), false));
tokens.Add(new BasicToken(TokenType.LineBreak, position));
file.Move(nextLine);
}
file.Reset(); // 重置游标至-1
if (tokens.Count == 0 || tokens.Last().Type != TokenType.LineBreak)
{
tokens.Add(new BasicToken(TokenType.LineBreak, position));
}
if (indent.Length <= 0)
{
return(tokens);
}
tokens.AddRange(indent.ShrinkTo(0).Select(i => new BasicToken(TokenType.LeaveScope, position)));
tokens.Add(new BasicToken(TokenType.LineBreak, position));
return(tokens);
}
