public TextToken Peek()
{
if (next != null)
{
return(next);
}
next = GetNext();
return(next);
}
public TextToken GetNext()
{
if (next != null)
{
try
{
return(next);
}
finally
{
next = null;
}
}
SkipWhitespace();
int c = reader.Peek();
if (c == -1)
{
return new TextToken()
{
Type = TextTokenType.Eof
}
}
;
switch (c)
{
case '.':
case ',':
case ':':
case ';':
case '!':
case '?':
reader.Read();
return(new TextToken()
{
Type = TextTokenType.Sep, Text = ((char)c).ToString()
});
case '\r':
case '\n':
return(ParseNewLine());
default:
return(ParseWord());
}
}
private void FormatText()
{
const int MaxLineLength = 75;
List <string> lines = new List <string>();
SimpleWordLexer lexer = new SimpleWordLexer(Result);
StringBuilder currentLine = new StringBuilder();
for (TextToken token = lexer.GetNext();
token.Type != TextTokenType.Eof;
token = lexer.GetNext())
{
if (token.Type == TextTokenType.Newline)
{
TextToken next = lexer.Peek();
if (next.Type == TextTokenType.Newline)
{
lines.Add(currentLine.ToString());
lines.Add(string.Empty);
currentLine.Clear();
token = lexer.GetNext();
}
}
if (token.Type == TextTokenType.Word)
{
var next = lexer.Peek();
int addedRequiredSpace = 0;
if (currentLine.Length > 0)
{
addedRequiredSpace = 1;
}
if (next.Type == TextTokenType.Sep)
{
addedRequiredSpace = 2;
}
if (currentLine.Length + token.Text.Length + addedRequiredSpace >= MaxLineLength)
{
lines.Add(currentLine.ToString());
currentLine.Clear();
}
if (currentLine.Length > 0)
{
currentLine.Append(' ');
}
string textToAppend = token.Text;
while (textToAppend.Length > MaxLineLength)
{
lines.Add(textToAppend.Substring(0, MaxLineLength));
textToAppend = textToAppend.Substring(MaxLineLength);
currentLine.Clear();
}
currentLine.Append(textToAppend);
}
if (token.Type == TextTokenType.Sep)
{
currentLine.Append(token.Text);
}
}
if (currentLine.Length > 0)
{
lines.Add(currentLine.ToString());
}
this.lines = lines;
}