private ParseNode ParseTextOptional(TokenStream tokenStream, ref ParserState state) { if (tokenStream.Current.Symbol != TexSymbolKind.Text) { return(null); } tokenStream.ForceMoveNext(); switch (tokenStream.Current.Symbol) { case TexSymbolKind.GroupOpen: tokenStream.ForceMoveNext(); var sb = new StringBuilder(); while (tokenStream.Current.Symbol != TexSymbolKind.GroupClose) { sb.Append((char)tokenStream.Current.Value); tokenStream.ForceMoveNext(); } if (sb.Length == 0) { throw new TexParserException(tokenStream.Current, errorMessageTextValueEmpty); } tokenStream.ForceMoveNext(); return(new ParseNode(TexToken.FromValue(TexSymbolKind.Text, sb.ToString(), tokenStream.Current.SourcePosition, null))); default: throw new TexParserException(tokenStream.Current, new[] { TexSymbolKind.GroupOpen }); } }
public TokenStream Tokenise(TrackedTextReader reader) { char nextChar; while (reader.Peek() != -1) { nextChar = (char)reader.Peek(); if (char.IsWhiteSpace(nextChar)) { reader.Read(); } else if (char.IsDigit(nextChar)) { yield return(TexToken.FromNumber(ScanReal(reader), reader.Position, nextChar.ToString())); } else if (nextChar == '\\') { object value; var token = TexToken.FromValue(ScanLongSymbol(reader, out value), value, reader.Position, nextChar.ToString()); if (token.Symbol != TexSymbolKind.Unknown) { yield return(token); } } else { object value; yield return(TexToken.FromValue(ScanShortSymbol(reader, out value), value, reader.Position, nextChar.ToString())); } } yield return(TexToken.FromSymbol(TexSymbolKind.EndOfStream, reader.Position, null)); }
private void WriteValueNode(ComposedTokenStream tokenStream, TexExpressionNode node, ComposerState state) { tokenStream.Write(TexToken.FromValue(node.Symbol, node.Value)); }