/// <summary> /// 通过一个文件流解析,返回一系列token /// </summary> /// <param name="pSourceTextStream"></param> /// <returns></returns> public List <Token> Parser(SourceTextStream pSourceTextStream) { if (pSourceTextStream == null) { return(null); } if (m_terminals == null || m_terminals.Count < 0) { return(null); } List <Token> l_tokens = new List <Token>(); int l_stackOverLine = 100000000; int l_cycleCount = 0; do { Token l_token = null; //空格跳过 while (pSourceTextStream.Cur() == ' ') { pSourceTextStream.Next(); } //根据终端列表切割token,当第一个切割成功,则至下一个 for (int i = 0; i < m_terminals.Count; i++) { Terminal l_terminal = m_terminals[i]; //标记读取位置,如果匹配成功,则无视这个位置 pSourceTextStream.MarkPos(); if (l_terminal.Match(pSourceTextStream)) { l_token = l_terminal.CutOffToken(pSourceTextStream); //如果token值为空值,则出现异常,直接返回 if (string.IsNullOrEmpty(l_token.Value)) { return(l_tokens); } l_tokens.Add(l_token); break; } //如果匹配失败,则退回到标记位置 pSourceTextStream.RevertPos(); } if (l_token == null) { break; } l_cycleCount++; } while (!pSourceTextStream.EndOfFile() || l_cycleCount > l_stackOverLine); return(l_tokens); }
public void MinimumLength() { var sourceText = SourceText.From("hello world", s_utf8NoBom); using (var stream = new SourceTextStream(sourceText)) { var buffer = new byte[100]; var max = s_utf8NoBom.GetMaxByteCount(charCount: 1); for (int i = 0; i < max; i++) { var local = i; Assert.Throws(typeof(ArgumentException), () => stream.Read(buffer, 0, local)); } } }
public void Issue1197() { var baseText = "food time"; var text = string.Format("{0}{1}", baseText, '\u2019'); var encoding = s_utf8NoBom; var sourceText = SourceText.From(text, encoding); using (var stream = new SourceTextStream(sourceText, bufferSize: text.Length * 2)) { var buffer = new byte[baseText.Length + 1]; Assert.Equal(baseText.Length, stream.Read(buffer, 0, buffer.Length)); Assert.True(buffer.Take(baseText.Length).SequenceEqual(encoding.GetBytes(baseText))); Assert.Equal(3, stream.Read(buffer, 0, buffer.Length)); Assert.True(buffer.Take(3).SequenceEqual(encoding.GetBytes(new[] { '\u2019' }))); } }
public void Parser() { Stream l_stream; try { l_stream = new FileStream(m_filePath, FileMode.Open); } catch (Exception) { throw; } StreamReader l_reader = new StreamReader(l_stream); m_sourceTextStream = new SourceTextStream(l_reader.ReadToEnd()); m_tokens = TokenParser.Instance.Parser(m_sourceTextStream); List <Token> l_tokens = m_tokens; }
public override Token CutOffToken(SourceTextStream pSourceTextStream) { Token l_token; char l_char = pSourceTextStream.Cur(); StringBuilder l_builder = new StringBuilder(); while (!CharMatch(l_char) && l_char != '\0') { l_builder.Append(l_char); l_char = pSourceTextStream.Next(); } //多种匹配,下一个字符如果还是匹配,则继续下一个字符,多种结合在一起 while (CharMatch(l_char)) { l_char = pSourceTextStream.Next(); } l_token = new Token(TokenCategory.EOL, l_builder.ToString()); return(l_token); }
public override bool Match(SourceTextStream pSourceTextStream) { return(true); }
/// <summary> /// 从字符流中截取一个token,同时字符流标志位向后移动 /// </summary> /// <param name="pSourceTextStream"></param> /// <returns></returns> public abstract Token CutOffToken(SourceTextStream pSourceTextStream);
public abstract bool Match(SourceTextStream pSourceTextStream);
public override Token CutOffToken(SourceTextStream pSourceTextStream) { Token l_token = new Token(); return(l_token); }