public override Token CutOffToken(SourceTextStream pSourceTextStream)
{
Token l_token;
char l_char = pSourceTextStream.Cur();
StringBuilder l_builder = new StringBuilder();
while (!CharMatch(l_char) && l_char != '\0')
{
l_builder.Append(l_char);
l_char = pSourceTextStream.Next();
}
//多种匹配，下一个字符如果还是匹配，则继续下一个字符，多种结合在一起
while (CharMatch(l_char))
{
l_char = pSourceTextStream.Next();
}
l_token = new Token(TokenCategory.EOL, l_builder.ToString());
return(l_token);
}
/// <summary>
/// 通过一个文件流解析，返回一系列token
/// </summary>
/// <param name="pSourceTextStream"></param>
/// <returns></returns>
public List <Token> Parser(SourceTextStream pSourceTextStream)
{
if (pSourceTextStream == null)
{
return(null);
}
if (m_terminals == null || m_terminals.Count < 0)
{
return(null);
}
List <Token> l_tokens = new List <Token>();
int l_stackOverLine = 100000000;
int l_cycleCount = 0;
do
{
Token l_token = null;
//空格跳过
while (pSourceTextStream.Cur() == ' ')
{
pSourceTextStream.Next();
}
//根据终端列表切割token，当第一个切割成功，则至下一个
for (int i = 0; i < m_terminals.Count; i++)
{
Terminal l_terminal = m_terminals[i];
//标记读取位置，如果匹配成功,则无视这个位置
pSourceTextStream.MarkPos();
if (l_terminal.Match(pSourceTextStream))
{
l_token = l_terminal.CutOffToken(pSourceTextStream);
//如果token值为空值，则出现异常，直接返回
if (string.IsNullOrEmpty(l_token.Value))
{
return(l_tokens);
}
l_tokens.Add(l_token);
break;
}
//如果匹配失败，则退回到标记位置
pSourceTextStream.RevertPos();
}
if (l_token == null)
{
break;
}
l_cycleCount++;
} while (!pSourceTextStream.EndOfFile() || l_cycleCount > l_stackOverLine);
return(l_tokens);
}