public static IEnumerable <LineToken> GetTokens(string input, bool treatWhitespaceOnlyLinesAsBlankLines)
{
if (input is null)
{
throw new ArgumentNullException(nameof(input));
}
var currentLine = new StringBuilder();
var currentLineIsWhitespace = true;
var lineNumber = 1;
LineToken GetCurrentToken()
{
// In "Loose Mode", lines that consist only of whitespace characters are considered blank lines
if (currentLine.Length == 0 || (treatWhitespaceOnlyLinesAsBlankLines && currentLineIsWhitespace))
{
// clear current value of line
currentLine.Clear();
return(LineToken.Blank(lineNumber++));
}
else
{
// get current value and reset state
var value = currentLine.GetValueAndClear();
currentLineIsWhitespace = true;
return(LineToken.Line(value, lineNumber++));
}
}
for (var i = 0; i < input.Length; i++)
{
var currentChar = input[i];
var nextChar = i + 1 < input.Length ? (char?)input[i + 1] : null;
switch ((currentChar, nextChar))
{
case ('\r', '\n'):
yield return(GetCurrentToken());
i += 1;
break;
case ('\n', _):
yield return(GetCurrentToken());
break;
default:
currentLineIsWhitespace &= char.IsWhiteSpace(currentChar);
currentLine.Append(currentChar);
break;
}
}
if (currentLine.Length > 0)
{
yield return(GetCurrentToken());
}
yield return(LineToken.Eof(lineNumber));
}
static object[] TestCase(string input, params FooterToken[] tokens) =>
new object[]
{
new XunitSerializableLineToken(LineToken.Line(input, 1)),
tokens.Select(t => new XunitSerializableFooterToken(t)).ToArray()
};