/// <summary>
/// 通过一个文件流解析，返回一系列token
/// </summary>
/// <param name="pSourceTextStream"></param>
/// <returns></returns>
public List <Token> Parser(SourceTextStream pSourceTextStream)
{
if (pSourceTextStream == null)
{
return(null);
}
if (m_terminals == null || m_terminals.Count < 0)
{
return(null);
}
List <Token> l_tokens = new List <Token>();
int l_stackOverLine = 100000000;
int l_cycleCount = 0;
do
{
Token l_token = null;
//空格跳过
while (pSourceTextStream.Cur() == ' ')
{
pSourceTextStream.Next();
}
//根据终端列表切割token，当第一个切割成功，则至下一个
for (int i = 0; i < m_terminals.Count; i++)
{
Terminal l_terminal = m_terminals[i];
//标记读取位置，如果匹配成功,则无视这个位置
pSourceTextStream.MarkPos();
if (l_terminal.Match(pSourceTextStream))
{
l_token = l_terminal.CutOffToken(pSourceTextStream);
//如果token值为空值，则出现异常，直接返回
if (string.IsNullOrEmpty(l_token.Value))
{
return(l_tokens);
}
l_tokens.Add(l_token);
break;
}
//如果匹配失败，则退回到标记位置
pSourceTextStream.RevertPos();
}
if (l_token == null)
{
break;
}
l_cycleCount++;
} while (!pSourceTextStream.EndOfFile() || l_cycleCount > l_stackOverLine);
return(l_tokens);
}
public void MinimumLength()
{
var sourceText = SourceText.From("hello world", s_utf8NoBom);
using (var stream = new SourceTextStream(sourceText))
{
var buffer = new byte[100];
var max = s_utf8NoBom.GetMaxByteCount(charCount: 1);
for (int i = 0; i < max; i++)
{
var local = i;
Assert.Throws(typeof(ArgumentException), () => stream.Read(buffer, 0, local));
}
}
}
public void Issue1197()
{
var baseText = "food time";
var text = string.Format("{0}{1}", baseText, '\u2019');
var encoding = s_utf8NoBom;
var sourceText = SourceText.From(text, encoding);
using (var stream = new SourceTextStream(sourceText, bufferSize: text.Length * 2))
{
var buffer = new byte[baseText.Length + 1];
Assert.Equal(baseText.Length, stream.Read(buffer, 0, buffer.Length));
Assert.True(buffer.Take(baseText.Length).SequenceEqual(encoding.GetBytes(baseText)));
Assert.Equal(3, stream.Read(buffer, 0, buffer.Length));
Assert.True(buffer.Take(3).SequenceEqual(encoding.GetBytes(new[] { '\u2019' })));
}
}
public void Parser()
{
Stream l_stream;
try
{
l_stream = new FileStream(m_filePath, FileMode.Open);
}
catch (Exception)
{
throw;
}
StreamReader l_reader = new StreamReader(l_stream);
m_sourceTextStream = new SourceTextStream(l_reader.ReadToEnd());
m_tokens = TokenParser.Instance.Parser(m_sourceTextStream);
List <Token> l_tokens = m_tokens;
}
public override Token CutOffToken(SourceTextStream pSourceTextStream)
{
Token l_token;
char l_char = pSourceTextStream.Cur();
StringBuilder l_builder = new StringBuilder();
while (!CharMatch(l_char) && l_char != '\0')
{
l_builder.Append(l_char);
l_char = pSourceTextStream.Next();
}
//多种匹配，下一个字符如果还是匹配，则继续下一个字符，多种结合在一起
while (CharMatch(l_char))
{
l_char = pSourceTextStream.Next();
}
l_token = new Token(TokenCategory.EOL, l_builder.ToString());
return(l_token);
}
public override bool Match(SourceTextStream pSourceTextStream)
{
return(true);
}
/// <summary> /// 从字符流中截取一个token，同时字符流标志位向后移动 /// </summary> /// <param name="pSourceTextStream"></param> /// <returns></returns> public abstract Token CutOffToken(SourceTextStream pSourceTextStream);
public abstract bool Match(SourceTextStream pSourceTextStream);
public override Token CutOffToken(SourceTextStream pSourceTextStream)
{
Token l_token = new Token();
return(l_token);
}
